diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 71a0f37aaf..81908e4a1f 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -80,11 +80,24 @@ string(REGEX REPLACE "${protobuf_VERSION_REGEX}" "\\5" message(STATUS "${protobuf_VERSION_PRERELEASE}") +message(STATUS "${protobuf_VERSION_PRERELEASE}") + # Package version set(protobuf_VERSION "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}") if(protobuf_VERSION_PRERELEASE) +<<<<<<< +======= + set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}") +else() + set(protobuf_VERSION "${protobuf_VERSION}.0") +endif() +message(STATUS "${protobuf_VERSION}") + +if(protobuf_VERBOSE) + +>>>>>>> set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}") else() set(protobuf_VERSION "${protobuf_VERSION}.0") diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index 9b429f7c52..8456d015cb 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -115,7 +115,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h" copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h -copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite_inl.h" include\google\protobuf\wire_format_lite_inl.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 271476dc5e..54da406c3c 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -119,6 +119,10 @@ message ConformanceRequest { // Specify details for how to encode jspb. JspbEncodingConfig jspb_encoding_options = 6; + + // This can be used in json and text format. If true, testee should print + // unknown fields instead of ignore. This feature is optional. + bool print_unknown_fields = 9; } // Represents a single test case's output. diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index ff70d5d9b9..5a1f21404d 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -214,8 +214,10 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { } case conformance::TEXT_FORMAT: { - GOOGLE_CHECK(TextFormat::PrintToString(*test_message, - response->mutable_text_payload())); + TextFormat::Printer printer; + printer.SetHideUnknownFields(!request.print_unknown_fields()); + GOOGLE_CHECK(printer.PrintToString(*test_message, + response->mutable_text_payload())); break; } diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index 5c4f900995..88d9749389 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -166,7 +166,8 @@ def do_test(request): return response elif request.requested_output_format == conformance_pb2.TEXT_FORMAT: - response.text_payload = text_format.MessageToString(test_message) + response.text_payload = text_format.MessageToString( + test_message, print_unknown_fields=request.print_unknown_fields) except Exception as e: response.runtime_error = str(e) diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 221464c031..6325b354a3 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -68,6 +68,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( input_format_(input_format), output_format_(output_format), prototype_message_(prototype_message), + prototype_message_for_compare_(prototype_message.New()), test_name_(test_name) { switch (input_format) { case conformance::PROTOBUF: { @@ -102,7 +103,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( Message* ConformanceTestSuite::ConformanceRequestSetting:: GetTestMessage() const { - return prototype_message_.New(); + return prototype_message_for_compare_->New(); } string ConformanceTestSuite::ConformanceRequestSetting:: diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index b64943dd53..4d741e7f15 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -224,6 +224,14 @@ class ConformanceTestSuite { string ConformanceLevelToString(ConformanceLevel level) const; + void SetPrintUnknownFields(bool print_unknown_fields) { + request_.set_print_unknown_fields(true); + } + + void SetPrototypeMessageForCompare(const Message& message) { + prototype_message_for_compare_.reset(message.New()); + } + protected: virtual string InputFormatString(conformance::WireFormat format) const; virtual string OutputFormatString(conformance::WireFormat format) const; @@ -234,6 +242,7 @@ class ConformanceTestSuite { ::conformance::WireFormat input_format_; ::conformance::WireFormat output_format_; const Message& prototype_message_; + std::unique_ptr prototype_message_for_compare_; string test_name_; }; diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc index 17af09e3e8..76f398c1a5 100644 --- a/conformance/text_format_conformance_suite.cc +++ b/conformance/text_format_conformance_suite.cc @@ -43,6 +43,7 @@ using conformance::WireFormat; using google::protobuf::Message; using google::protobuf::TextFormat; using protobuf_test_messages::proto2::TestAllTypesProto2; +using protobuf_test_messages::proto2::UnknownToTestAllTypes; using protobuf_test_messages::proto3::TestAllTypesProto3; using std::string; @@ -54,8 +55,14 @@ TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() { } bool TextFormatConformanceTestSuite::ParseTextFormatResponse( - const ConformanceResponse& response, Message* test_message) { - if (!TextFormat::ParseFromString(response.text_payload(), test_message)) { + const ConformanceResponse& response, + const ConformanceRequestSetting& setting, Message* test_message) { + TextFormat::Parser parser; + const ConformanceRequest& request = setting.GetRequest(); + if (request.print_unknown_fields()) { + parser.AllowFieldNumber(true); + } + if (!parser.ParseFromString(response.text_payload(), test_message)) { GOOGLE_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode " << "yielded unparseable proto. Text payload: " << response.text_payload(); @@ -103,7 +110,7 @@ bool TextFormatConformanceTestSuite::ParseResponse( return false; } - if (!ParseTextFormatResponse(response, test_message)) { + if (!ParseTextFormatResponse(response, setting, test_message)) { ReportFailure( test_name, level, request, response, "TEXT_FORMAT output we received from test was unparseable."); @@ -171,6 +178,27 @@ void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage( RunValidInputTest(setting2, input_text); } +void TextFormatConformanceTestSuite::RunValidUnknownTextFormatTest( + const string& test_name, const Message& message) { + string serialized_input; + message.SerializeToString(&serialized_input); + TestAllTypesProto3 prototype; + ConformanceRequestSetting setting1( + RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT, + conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Drop", + serialized_input); + setting1.SetPrototypeMessageForCompare(message); + RunValidBinaryInputTest(setting1, ""); + + ConformanceRequestSetting setting2( + RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT, + conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Print", + serialized_input); + setting2.SetPrototypeMessageForCompare(message); + setting2.SetPrintUnknownFields(true); + RunValidBinaryInputTest(setting2, serialized_input); +} + void TextFormatConformanceTestSuite::RunSuiteImpl() { RunValidTextFormatTest("HelloWorld", REQUIRED, "optional_string: 'Hello, World!'"); @@ -235,6 +263,29 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() { "Data: { group_int32: 1 }"); RunValidTextFormatTestProto2("GroupFieldEmpty", REQUIRED, "Data {}"); + + + // Unknown Fields + UnknownToTestAllTypes message; + // Unable to print unknown Fixed32/Fixed64 fields as if they are known. + // Fixed32/Fixed64 fields are not added in the tests. + message.set_optional_int32(123); + message.set_optional_string("hello"); + message.set_optional_bool(true); + RunValidUnknownTextFormatTest("ScalarUnknownFields", message); + + message.Clear(); + message.mutable_nested_message()->set_c(111); + RunValidUnknownTextFormatTest("MessageUnknownFields", message); + + message.Clear(); + message.mutable_optionalgroup()->set_a(321); + RunValidUnknownTextFormatTest("GroupUnknownFields", message); + + message.add_repeated_int32(1); + message.add_repeated_int32(2); + message.add_repeated_int32(3); + RunValidUnknownTextFormatTest("RepeatedUnknownFields", message); } } // namespace protobuf diff --git a/conformance/text_format_conformance_suite.h b/conformance/text_format_conformance_suite.h index 661d45e095..dd258f50b8 100644 --- a/conformance/text_format_conformance_suite.h +++ b/conformance/text_format_conformance_suite.h @@ -51,9 +51,12 @@ class TextFormatConformanceTestSuite : public ConformanceTestSuite { ConformanceLevel level, const string& input_text, const Message& prototype); + void RunValidUnknownTextFormatTest(const string& test_name, + const Message& message); void ExpectParseFailure(const string& test_name, ConformanceLevel level, const string& input); bool ParseTextFormatResponse(const conformance::ConformanceResponse& response, + const ConformanceRequestSetting& setting, Message* test_message); bool ParseResponse(const conformance::ConformanceResponse& response, const ConformanceRequestSetting& setting, diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt new file mode 100755 index 0000000000..4902d461cd --- /dev/null +++ b/conformance/text_format_failure_list_java.txt @@ -0,0 +1,4 @@ +Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput +Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 54d158d4a9..648991dc25 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -1438,16 +1438,14 @@ public abstract class ByteString implements Iterable, Serializable { LiteralByteString lbsOther = (LiteralByteString) other; byte[] thisBytes = bytes; byte[] otherBytes = lbsOther.bytes; - int thisLimit = getOffsetIntoBytes() + length; - for (int thisIndex = getOffsetIntoBytes(), - otherIndex = lbsOther.getOffsetIntoBytes() + offset; - (thisIndex < thisLimit); - ++thisIndex, ++otherIndex) { - if (thisBytes[thisIndex] != otherBytes[otherIndex]) { - return false; - } - } - return true; + + return UnsafeUtil.mismatch( + thisBytes, + getOffsetIntoBytes(), + otherBytes, + lbsOther.getOffsetIntoBytes() + offset, + length) + == -1; } return other.substring(offset, offset + length).equals(substring(0, length)); diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index 53771de86d..3823f81fc7 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -1264,16 +1264,34 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public final void writeUInt32NoTag(int value) throws IOException { - if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT32_SIZE) { - while (true) { - if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(buffer, position++, (byte) value); - return; - } else { - UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } + if (HAS_UNSAFE_ARRAY_OPERATIONS + && !Android.isOnAndroidDevice() + && spaceLeft() >= MAX_VARINT32_SIZE) { + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + return; + } + UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80)); + value >>>= 7; + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + return; + } + UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80)); + value >>>= 7; + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + return; + } + UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80)); + value >>>= 7; + if ((value & ~0x7F) == 0) { + UnsafeUtil.putByte(buffer, position++, (byte) value); + return; } + UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80)); + value >>>= 7; + UnsafeUtil.putByte(buffer, position++, (byte) value); } else { try { while (true) { diff --git a/java/core/src/main/java/com/google/protobuf/Extension.java b/java/core/src/main/java/com/google/protobuf/Extension.java index e5da634f1c..30c828e57b 100644 --- a/java/core/src/main/java/com/google/protobuf/Extension.java +++ b/java/core/src/main/java/com/google/protobuf/Extension.java @@ -30,6 +30,7 @@ package com.google.protobuf; +// TODO(chrisn): Change ContainingType to extend Message /** * Interface that generated extensions implement. * @@ -37,6 +38,11 @@ package com.google.protobuf; */ public abstract class Extension extends ExtensionLite { + // TODO(chrisn): Add package-private constructor. + + /** {@inheritDoc} Overridden to return {@link Message} instead of {@link MessageLite}. */ + @Override + public abstract Message getMessageDefaultInstance(); /** Returns the descriptor of the extension. */ public abstract Descriptors.FieldDescriptor getDescriptor(); diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 77c837e6fd..e16633baec 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -448,22 +448,26 @@ public class JsonFormat { } /** - * Find a type by its full name. Returns null if it cannot be found in - * this {@link TypeRegistry}. + * Find a type by its full name. Returns null if it cannot be found in this {@link + * TypeRegistry}. */ public Descriptor find(String name) { return types.get(name); } + /* @Nullable */ + Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException { + return find(getTypeName(typeUrl)); + } + private final Map types; private TypeRegistry(Map types) { this.types = types; } - /** - * A Builder is used to build {@link TypeRegistry}. - */ + + /** A Builder is used to build {@link TypeRegistry}. */ public static class Builder { private Builder() {} @@ -801,15 +805,14 @@ public class JsonFormat { throw new InvalidProtocolBufferException("Invalid Any type."); } String typeUrl = (String) message.getField(typeUrlField); - String typeName = getTypeName(typeUrl); - Descriptor type = registry.find(typeName); + Descriptor type = registry.getDescriptorForTypeUrl(typeUrl); if (type == null) { throw new InvalidProtocolBufferException("Cannot find type for url: " + typeUrl); } ByteString content = (ByteString) message.getField(valueField); Message contentMessage = DynamicMessage.getDefaultInstance(type).getParserForType().parseFrom(content); - WellKnownTypePrinter printer = wellKnownTypePrinters.get(typeName); + WellKnownTypePrinter printer = wellKnownTypePrinters.get(getTypeName(typeUrl)); if (printer != null) { // If the type is one of the well-known types, we use a special // formatting. @@ -1443,7 +1446,7 @@ public class JsonFormat { throw new InvalidProtocolBufferException("Missing type url when parsing: " + json); } String typeUrl = typeUrlElement.getAsString(); - Descriptor contentType = registry.find(getTypeName(typeUrl)); + Descriptor contentType = registry.getDescriptorForTypeUrl(typeUrl); if (contentType == null) { throw new InvalidProtocolBufferException("Cannot resolve type: " + typeUrl); } diff --git a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java index 922532a9d2..0c19ad5eb5 100644 --- a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java +++ b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java @@ -112,8 +112,9 @@ public final class Timestamps { }; /** - * Returns a {@link Comparator} for {@link Timestamp}s which sorts in increasing chronological - * order. Nulls and invalid {@link Timestamp}s are not allowed (see {@link #isValid}). + * Returns a {@link Comparator} for {@link Timestamp Timestamps} which sorts in increasing + * chronological order. Nulls and invalid {@link Timestamp Timestamps} are not allowed (see + * {@link #isValid}). */ public static Comparator comparator() { return COMPARATOR; diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index 9805737b88..b7b437ceac 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -34,6 +34,7 @@ import com.google.protobuf.Any; import com.google.protobuf.BoolValue; import com.google.protobuf.ByteString; import com.google.protobuf.BytesValue; +import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DoubleValue; import com.google.protobuf.FloatValue; @@ -834,6 +835,7 @@ public class JsonFormatTest extends TestCase { assertRoundTripEquals(message); } + public void testAnyFields() throws Exception { TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build(); TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build(); diff --git a/js/data.proto b/js/data.proto index 74a8a994c7..ca815caf61 100644 --- a/js/data.proto +++ b/js/data.proto @@ -32,11 +32,11 @@ syntax = "proto2"; +package jspb.test; + option java_package = "com.google.apps.jspb.proto"; option java_multiple_files = true; -package jspb.test; - // legacy data, must be nested message data { message NestedData { @@ -48,4 +48,3 @@ message data { message UnnestedData { required string str = 1; } - diff --git a/js/proto3_test.proto b/js/proto3_test.proto index 0d073ea03e..f23e19c9d8 100644 --- a/js/proto3_test.proto +++ b/js/proto3_test.proto @@ -35,43 +35,43 @@ import "testbinary.proto"; package jspb.test; message TestProto3 { - int32 optional_int32 = 1; - int64 optional_int64 = 2; - uint32 optional_uint32 = 3; - uint64 optional_uint64 = 4; - sint32 optional_sint32 = 5; - sint64 optional_sint64 = 6; - fixed32 optional_fixed32 = 7; - fixed64 optional_fixed64 = 8; - sfixed32 optional_sfixed32 = 9; + int32 optional_int32 = 1; + int64 optional_int64 = 2; + uint32 optional_uint32 = 3; + uint64 optional_uint64 = 4; + sint32 optional_sint32 = 5; + sint64 optional_sint64 = 6; + fixed32 optional_fixed32 = 7; + fixed64 optional_fixed64 = 8; + sfixed32 optional_sfixed32 = 9; sfixed64 optional_sfixed64 = 10; - float optional_float = 11; - double optional_double = 12; - bool optional_bool = 13; - string optional_string = 14; - bytes optional_bytes = 15; + float optional_float = 11; + double optional_double = 12; + bool optional_bool = 13; + string optional_string = 14; + bytes optional_bytes = 15; ForeignMessage optional_foreign_message = 19; - Proto3Enum optional_foreign_enum = 22; + Proto3Enum optional_foreign_enum = 22; - repeated int32 repeated_int32 = 31; - repeated int64 repeated_int64 = 32; - repeated uint32 repeated_uint32 = 33; - repeated uint64 repeated_uint64 = 34; - repeated sint32 repeated_sint32 = 35; - repeated sint64 repeated_sint64 = 36; - repeated fixed32 repeated_fixed32 = 37; - repeated fixed64 repeated_fixed64 = 38; + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; repeated sfixed32 repeated_sfixed32 = 39; repeated sfixed64 repeated_sfixed64 = 40; - repeated float repeated_float = 41; - repeated double repeated_double = 42; - repeated bool repeated_bool = 43; - repeated string repeated_string = 44; - repeated bytes repeated_bytes = 45; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; repeated ForeignMessage repeated_foreign_message = 49; - repeated Proto3Enum repeated_foreign_enum = 52; + repeated Proto3Enum repeated_foreign_enum = 52; oneof oneof_field { diff --git a/js/test13.proto b/js/test13.proto index 4f9d272b60..b9895d87ae 100644 --- a/js/test13.proto +++ b/js/test13.proto @@ -47,24 +47,24 @@ enum TestEnum { message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName1 { optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName2 - a = 1; + a = 1; optional int32 b = 2; } message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName2 { optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName3 - a = 1; + a = 1; optional int32 b = 2; } message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName3 { optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName4 - a = 1; + a = 1; optional int32 b = 2; } message TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName4 { optional TestLooooooooooooooooooooooooooooooooooooooooooooooooooooooongName1 - a = 1; + a = 1; optional int32 b = 2; } diff --git a/js/test15.proto b/js/test15.proto index 602cc2da4d..b481116519 100644 --- a/js/test15.proto +++ b/js/test15.proto @@ -30,10 +30,10 @@ syntax = "proto2"; -import "test13.proto"; - package jspb.filenametest.package1; +import "test13.proto"; + extend TestMessage { optional int32 b = 2; } diff --git a/js/test2.proto b/js/test2.proto index b67f93fa9e..e9457e7c71 100644 --- a/js/test2.proto +++ b/js/test2.proto @@ -30,13 +30,13 @@ syntax = "proto2"; -option java_package = "com.google.apps.jspb.proto"; -option java_multiple_files = true; - package jspb.test; import "test.proto"; +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + message TestExtensionsMessage { optional int32 intfield = 1; extensions 100 to max; diff --git a/js/test3.proto b/js/test3.proto index 940a552ed5..3fa037dfba 100644 --- a/js/test3.proto +++ b/js/test3.proto @@ -30,11 +30,11 @@ syntax = "proto2"; +package jspb.exttest; + option java_package = "com.google.apps.jspb.proto"; option java_multiple_files = true; -package jspb.exttest; - message TestExtensionsMessage { optional int32 intfield = 1; extensions 100 to max; diff --git a/js/test4.proto b/js/test4.proto index cf2451e9cb..c3c8342411 100644 --- a/js/test4.proto +++ b/js/test4.proto @@ -30,13 +30,13 @@ syntax = "proto2"; -option java_package = "com.google.apps.jspb.proto"; -option java_multiple_files = true; - package jspb.exttest; import "test3.proto"; +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + extend TestExtensionsMessage { optional ExtensionMessage floating_msg_field_two = 103; } diff --git a/js/test5.proto b/js/test5.proto index 34979517e5..db297213a4 100644 --- a/js/test5.proto +++ b/js/test5.proto @@ -30,11 +30,11 @@ syntax = "proto2"; +package jspb.exttest.beta; + option java_package = "com.google.apps.jspb.proto"; option java_multiple_files = true; -package jspb.exttest.beta; - message TestBetaExtensionsMessage { extensions 100 to max; } diff --git a/js/test8.proto b/js/test8.proto index 2ae80dab67..7dbb6eff14 100644 --- a/js/test8.proto +++ b/js/test8.proto @@ -30,11 +30,11 @@ syntax = "proto2"; +package jspb.exttest.nested; + option java_package = "com.google.apps.jspb.proto"; option java_multiple_files = true; -package jspb.exttest.nested; - message TestNestedExtensionsMessage { optional int32 intfield = 1; extensions 100 to max; diff --git a/js/testbinary.proto b/js/testbinary.proto index ee4d2dfac4..2e548454da 100644 --- a/js/testbinary.proto +++ b/js/testbinary.proto @@ -39,66 +39,66 @@ package jspb.test; // to ensure that the binary-format support will handle all field types // properly. message TestAllTypes { - optional int32 optional_int32 = 1; - optional int64 optional_int64 = 2; - optional uint32 optional_uint32 = 3; - optional uint64 optional_uint64 = 4; - optional sint32 optional_sint32 = 5; - optional sint64 optional_sint64 = 6; - optional fixed32 optional_fixed32 = 7; - optional fixed64 optional_fixed64 = 8; - optional sfixed32 optional_sfixed32 = 9; + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional sint32 optional_sint32 = 5; + optional sint64 optional_sint64 = 6; + optional fixed32 optional_fixed32 = 7; + optional fixed64 optional_fixed64 = 8; + optional sfixed32 optional_sfixed32 = 9; optional sfixed64 optional_sfixed64 = 10; - optional float optional_float = 11; - optional double optional_double = 12; - optional bool optional_bool = 13; - optional string optional_string = 14; - optional bytes optional_bytes = 15; + optional float optional_float = 11; + optional double optional_double = 12; + optional bool optional_bool = 13; + optional string optional_string = 14; + optional bytes optional_bytes = 15; optional group OptionalGroup = 16 { optional int32 a = 17; } - optional ForeignMessage optional_foreign_message = 19; - optional ForeignEnum optional_foreign_enum = 22; + optional ForeignMessage optional_foreign_message = 19; + optional ForeignEnum optional_foreign_enum = 22; // Repeated - repeated int32 repeated_int32 = 31; - repeated int64 repeated_int64 = 32; - repeated uint32 repeated_uint32 = 33; - repeated uint64 repeated_uint64 = 34; - repeated sint32 repeated_sint32 = 35; - repeated sint64 repeated_sint64 = 36; - repeated fixed32 repeated_fixed32 = 37; - repeated fixed64 repeated_fixed64 = 38; + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; repeated sfixed32 repeated_sfixed32 = 39; repeated sfixed64 repeated_sfixed64 = 40; - repeated float repeated_float = 41; - repeated double repeated_double = 42; - repeated bool repeated_bool = 43; - repeated string repeated_string = 44; - repeated bytes repeated_bytes = 45; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; repeated group RepeatedGroup = 46 { optional int32 a = 47; } - repeated ForeignMessage repeated_foreign_message = 49; - repeated ForeignEnum repeated_foreign_enum = 52; + repeated ForeignMessage repeated_foreign_message = 49; + repeated ForeignEnum repeated_foreign_enum = 52; // Packed repeated - repeated int32 packed_repeated_int32 = 61 [packed=true]; - repeated int64 packed_repeated_int64 = 62 [packed=true]; - repeated uint32 packed_repeated_uint32 = 63 [packed=true]; - repeated uint64 packed_repeated_uint64 = 64 [packed=true]; - repeated sint32 packed_repeated_sint32 = 65 [packed=true]; - repeated sint64 packed_repeated_sint64 = 66 [packed=true]; - repeated fixed32 packed_repeated_fixed32 = 67 [packed=true]; - repeated fixed64 packed_repeated_fixed64 = 68 [packed=true]; - repeated sfixed32 packed_repeated_sfixed32 = 69 [packed=true]; - repeated sfixed64 packed_repeated_sfixed64 = 70 [packed=true]; - repeated float packed_repeated_float = 71 [packed=true]; - repeated double packed_repeated_double = 72 [packed=true]; - repeated bool packed_repeated_bool = 73 [packed=true]; + repeated int32 packed_repeated_int32 = 61 [packed = true]; + repeated int64 packed_repeated_int64 = 62 [packed = true]; + repeated uint32 packed_repeated_uint32 = 63 [packed = true]; + repeated uint64 packed_repeated_uint64 = 64 [packed = true]; + repeated sint32 packed_repeated_sint32 = 65 [packed = true]; + repeated sint64 packed_repeated_sint64 = 66 [packed = true]; + repeated fixed32 packed_repeated_fixed32 = 67 [packed = true]; + repeated fixed64 packed_repeated_fixed64 = 68 [packed = true]; + repeated sfixed32 packed_repeated_sfixed32 = 69 [packed = true]; + repeated sfixed64 packed_repeated_sfixed64 = 70 [packed = true]; + repeated float packed_repeated_float = 71 [packed = true]; + repeated double packed_repeated_double = 72 [packed = true]; + repeated bool packed_repeated_bool = 73 [packed = true]; oneof oneof_field { uint32 oneof_uint32 = 111; @@ -132,55 +132,54 @@ message ExtendsWithMessage { } extend TestExtendable { - optional int32 extend_optional_int32 = 1; - optional int64 extend_optional_int64 = 2; - optional uint32 extend_optional_uint32 = 3; - optional uint64 extend_optional_uint64 = 4; - optional sint32 extend_optional_sint32 = 5; - optional sint64 extend_optional_sint64 = 6; - optional fixed32 extend_optional_fixed32 = 7; - optional fixed64 extend_optional_fixed64 = 8; - optional sfixed32 extend_optional_sfixed32 = 9; + optional int32 extend_optional_int32 = 1; + optional int64 extend_optional_int64 = 2; + optional uint32 extend_optional_uint32 = 3; + optional uint64 extend_optional_uint64 = 4; + optional sint32 extend_optional_sint32 = 5; + optional sint64 extend_optional_sint64 = 6; + optional fixed32 extend_optional_fixed32 = 7; + optional fixed64 extend_optional_fixed64 = 8; + optional sfixed32 extend_optional_sfixed32 = 9; optional sfixed64 extend_optional_sfixed64 = 10; - optional float extend_optional_float = 11; - optional double extend_optional_double = 12; - optional bool extend_optional_bool = 13; - optional string extend_optional_string = 14; - optional bytes extend_optional_bytes = 15; - optional ForeignEnum extend_optional_foreign_enum = 22; - - repeated int32 extend_repeated_int32 = 31; - repeated int64 extend_repeated_int64 = 32; - repeated uint32 extend_repeated_uint32 = 33; - repeated uint64 extend_repeated_uint64 = 34; - repeated sint32 extend_repeated_sint32 = 35; - repeated sint64 extend_repeated_sint64 = 36; - repeated fixed32 extend_repeated_fixed32 = 37; - repeated fixed64 extend_repeated_fixed64 = 38; + optional float extend_optional_float = 11; + optional double extend_optional_double = 12; + optional bool extend_optional_bool = 13; + optional string extend_optional_string = 14; + optional bytes extend_optional_bytes = 15; + optional ForeignEnum extend_optional_foreign_enum = 22; + + repeated int32 extend_repeated_int32 = 31; + repeated int64 extend_repeated_int64 = 32; + repeated uint32 extend_repeated_uint32 = 33; + repeated uint64 extend_repeated_uint64 = 34; + repeated sint32 extend_repeated_sint32 = 35; + repeated sint64 extend_repeated_sint64 = 36; + repeated fixed32 extend_repeated_fixed32 = 37; + repeated fixed64 extend_repeated_fixed64 = 38; repeated sfixed32 extend_repeated_sfixed32 = 39; repeated sfixed64 extend_repeated_sfixed64 = 40; - repeated float extend_repeated_float = 41; - repeated double extend_repeated_double = 42; - repeated bool extend_repeated_bool = 43; - repeated string extend_repeated_string = 44; - repeated bytes extend_repeated_bytes = 45; - repeated ForeignEnum extend_repeated_foreign_enum = 52; - - repeated int32 extend_packed_repeated_int32 = 61 [packed=true]; - repeated int64 extend_packed_repeated_int64 = 62 [packed=true]; - repeated uint32 extend_packed_repeated_uint32 = 63 [packed=true]; - repeated uint64 extend_packed_repeated_uint64 = 64 [packed=true]; - repeated sint32 extend_packed_repeated_sint32 = 65 [packed=true]; - repeated sint64 extend_packed_repeated_sint64 = 66 [packed=true]; - repeated fixed32 extend_packed_repeated_fixed32 = 67 [packed=true]; - repeated fixed64 extend_packed_repeated_fixed64 = 68 [packed=true]; - repeated sfixed32 extend_packed_repeated_sfixed32 = 69 [packed=true]; - repeated sfixed64 extend_packed_repeated_sfixed64 = 70 [packed=true]; - repeated float extend_packed_repeated_float = 71 [packed=true]; - repeated double extend_packed_repeated_double = 72 [packed=true]; - repeated bool extend_packed_repeated_bool = 73 [packed=true]; - repeated ForeignEnum extend_packed_repeated_foreign_enum = 82 - [packed=true]; + repeated float extend_repeated_float = 41; + repeated double extend_repeated_double = 42; + repeated bool extend_repeated_bool = 43; + repeated string extend_repeated_string = 44; + repeated bytes extend_repeated_bytes = 45; + repeated ForeignEnum extend_repeated_foreign_enum = 52; + + repeated int32 extend_packed_repeated_int32 = 61 [packed = true]; + repeated int64 extend_packed_repeated_int64 = 62 [packed = true]; + repeated uint32 extend_packed_repeated_uint32 = 63 [packed = true]; + repeated uint64 extend_packed_repeated_uint64 = 64 [packed = true]; + repeated sint32 extend_packed_repeated_sint32 = 65 [packed = true]; + repeated sint64 extend_packed_repeated_sint64 = 66 [packed = true]; + repeated fixed32 extend_packed_repeated_fixed32 = 67 [packed = true]; + repeated fixed64 extend_packed_repeated_fixed64 = 68 [packed = true]; + repeated sfixed32 extend_packed_repeated_sfixed32 = 69 [packed = true]; + repeated sfixed64 extend_packed_repeated_sfixed64 = 70 [packed = true]; + repeated float extend_packed_repeated_float = 71 [packed = true]; + repeated double extend_packed_repeated_double = 72 [packed = true]; + repeated bool extend_packed_repeated_bool = 73 [packed = true]; + repeated ForeignEnum extend_packed_repeated_foreign_enum = 82 [packed = true]; } @@ -226,7 +225,7 @@ message MapEntryOptionalKeysBoolKey { message TestMapFieldsOptionalKeys { optional MapEntryOptionalKeysStringKey map_string_string = 1; - optional MapEntryOptionalKeysInt32Key map_int32_string= 8; + optional MapEntryOptionalKeysInt32Key map_int32_string = 8; optional MapEntryOptionalKeysInt64Key map_int64_string = 9; optional MapEntryOptionalKeysBoolKey map_bool_string = 10; } diff --git a/js/testempty.proto b/js/testempty.proto index 960bce4e5c..6161753fd5 100644 --- a/js/testempty.proto +++ b/js/testempty.proto @@ -31,4 +31,3 @@ syntax = "proto2"; package javatests.com.google.apps.jspb; - diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 8a9ba3da48..5d8f8a1a5c 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -855,7 +855,7 @@ class FileDescriptor(DescriptorBase): dependencies: List of other FileDescriptors this FileDescriptor depends on. public_dependencies: A list of FileDescriptors, subset of the dependencies above, which were declared as "public". - message_types_by_name: Dict of message names of their descriptors. + message_types_by_name: Dict of message names and their descriptors. enum_types_by_name: Dict of enum names and their descriptors. extensions_by_name: Dict of extension names and their descriptors. services_by_name: Dict of services names and their descriptors. diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py index ca0bfecf7f..38f76c5ae2 100755 --- a/python/google/protobuf/internal/_parameterized.py +++ b/python/google/protobuf/internal/_parameterized.py @@ -145,7 +145,6 @@ be wrapped into a tuple: __author__ = 'tmarek@google.com (Torsten Marek)' -import collections import functools import re import types @@ -157,6 +156,13 @@ import uuid import six +try: + # Since python 3 + import collections.abc as collections_abc +except ImportError: + # Won't work after python 3.8 + import collections as collections_abc + ADDR_RE = re.compile(r'\<([a-zA-Z0-9_\-\.]+) object at 0x[a-fA-F0-9]+\>') _SEPARATOR = uuid.uuid1().hex _FIRST_ARG = object() @@ -174,12 +180,12 @@ def _StrClass(cls): def _NonStringIterable(obj): - return (isinstance(obj, collections.Iterable) and not + return (isinstance(obj, collections_abc.Iterable) and not isinstance(obj, six.string_types)) def _FormatParameterList(testcase_params): - if isinstance(testcase_params, collections.Mapping): + if isinstance(testcase_params, collections_abc.Mapping): return ', '.join('%s=%s' % (argname, _CleanRepr(value)) for argname, value in testcase_params.items()) elif _NonStringIterable(testcase_params): @@ -222,7 +228,7 @@ class _ParameterizedTestIter(object): def MakeBoundParamTest(testcase_params): @functools.wraps(test_method) def BoundParamTest(self): - if isinstance(testcase_params, collections.Mapping): + if isinstance(testcase_params, collections_abc.Mapping): test_method(self, **testcase_params) elif _NonStringIterable(testcase_params): test_method(self, *testcase_params) @@ -291,7 +297,7 @@ def _ParameterDecorator(naming_type, testcases): if isinstance(obj, type): _ModifyClass( obj, - list(testcases) if not isinstance(testcases, collections.Sequence) + list(testcases) if not isinstance(testcases, collections_abc.Sequence) else testcases, naming_type) return obj diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py index 5a5401841e..845d774274 100755 --- a/python/google/protobuf/internal/decoder.py +++ b/python/google/protobuf/internal/decoder.py @@ -914,11 +914,11 @@ def _SkipGroup(buffer, pos, end): pos = new_pos -def _DecodeGroup(buffer, pos): - """Decode group. Returns the UnknownFieldSet and new position.""" +def _DecodeUnknownFieldSet(buffer, pos, end_pos=None): + """Decode UnknownFieldSet. Returns the UnknownFieldSet and new position.""" unknown_field_set = containers.UnknownFieldSet() - while 1: + while end_pos is None or pos < end_pos: (tag_bytes, pos) = ReadTag(buffer, pos) (tag, _) = _DecodeVarint(tag_bytes, 0) field_number, wire_type = wire_format.UnpackTag(tag) @@ -945,7 +945,7 @@ def _DecodeUnknownField(buffer, pos, wire_type): data = buffer[pos:pos+size] pos += size elif wire_type == wire_format.WIRETYPE_START_GROUP: - (data, pos) = _DecodeGroup(buffer, pos) + (data, pos) = _DecodeUnknownFieldSet(buffer, pos) elif wire_type == wire_format.WIRETYPE_END_GROUP: return (0, -1) else: diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index da5dbd9252..800d54f324 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -44,10 +44,11 @@ from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 from google.protobuf.internal import factory_test2_pb2 from google.protobuf.internal import no_package_pb2 +from google.protobuf.internal import testing_refleaks from google.protobuf import descriptor_database -class DescriptorDatabaseTest(unittest.TestCase): +class DescriptorDatabaseTest(testing_refleaks.BaseTestCase): def testAdd(self): db = descriptor_database.DescriptorDatabase() diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 32442519ea..4dc209475d 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -55,6 +55,7 @@ from google.protobuf.internal import factory_test2_pb2 from google.protobuf.internal import file_options_test_pb2 from google.protobuf.internal import more_messages_pb2 from google.protobuf.internal import no_package_pb2 +from google.protobuf.internal import testing_refleaks from google.protobuf import descriptor from google.protobuf import descriptor_database from google.protobuf import descriptor_pool @@ -560,7 +561,8 @@ class DescriptorPoolTestBase(object): str(w[0].message)) -class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): +class DefaultDescriptorPoolTest(DescriptorPoolTestBase, + testing_refleaks.BaseTestCase): def setUp(self): self.pool = descriptor_pool.Default() @@ -595,7 +597,8 @@ class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): unittest_pb2.DESCRIPTOR.services_by_name['TestService']) -class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): +class CreateDescriptorPoolTest(DescriptorPoolTestBase, + testing_refleaks.BaseTestCase): def setUp(self): self.pool = descriptor_pool.DescriptorPool() @@ -617,7 +620,7 @@ class CreateDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase): class SecondaryDescriptorFromDescriptorDB(DescriptorPoolTestBase, - unittest.TestCase): + testing_refleaks.BaseTestCase): def setUp(self): self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( @@ -809,7 +812,7 @@ class ExtensionField(object): test.assertEqual(file_desc, field_desc.file) -class AddDescriptorTest(unittest.TestCase): +class AddDescriptorTest(testing_refleaks.BaseTestCase): def _TestMessage(self, prefix): pool = descriptor_pool.DescriptorPool() diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index 5be65c7851..7a5a090134 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -43,12 +43,13 @@ from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation from google.protobuf.internal import factory_test1_pb2 from google.protobuf.internal import factory_test2_pb2 +from google.protobuf.internal import testing_refleaks from google.protobuf import descriptor_database from google.protobuf import descriptor_pool from google.protobuf import message_factory -class MessageFactoryTest(unittest.TestCase): +class MessageFactoryTest(testing_refleaks.BaseTestCase): def setUp(self): self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index c3bb066f50..b66c1e0edb 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -45,7 +45,6 @@ abstract interface. __author__ = 'gps@google.com (Gregory P. Smith)' -import collections import copy import math import operator @@ -55,6 +54,13 @@ import six import sys import warnings +try: + # Since python 3 + import collections.abc as collections_abc +except ImportError: + # Won't work after python 3.8 + import collections as collections_abc + try: import unittest2 as unittest # PY26 except ImportError: @@ -753,9 +759,9 @@ class MessageTest(BaseTestCase): def testRepeatedFieldsAreSequences(self, message_module): m = message_module.TestAllTypes() - self.assertIsInstance(m.repeated_int32, collections.MutableSequence) + self.assertIsInstance(m.repeated_int32, collections_abc.MutableSequence) self.assertIsInstance(m.repeated_nested_message, - collections.MutableSequence) + collections_abc.MutableSequence) def testRepeatedFieldsNotHashable(self, message_module): m = message_module.TestAllTypes() @@ -2339,11 +2345,11 @@ class Proto3Test(BaseTestCase): def testMapsAreMapping(self): msg = map_unittest_pb2.TestMap() - self.assertIsInstance(msg.map_int32_int32, collections.Mapping) - self.assertIsInstance(msg.map_int32_int32, collections.MutableMapping) - self.assertIsInstance(msg.map_int32_foreign_message, collections.Mapping) + self.assertIsInstance(msg.map_int32_int32, collections_abc.Mapping) + self.assertIsInstance(msg.map_int32_int32, collections_abc.MutableMapping) + self.assertIsInstance(msg.map_int32_foreign_message, collections_abc.Mapping) self.assertIsInstance(msg.map_int32_foreign_message, - collections.MutableMapping) + collections_abc.MutableMapping) def testMapsCompare(self): msg = map_unittest_pb2.TestMap() diff --git a/python/google/protobuf/internal/test_bad_identifiers.proto b/python/google/protobuf/internal/test_bad_identifiers.proto index c4860ea88a..caf86b5f29 100644 --- a/python/google/protobuf/internal/test_bad_identifiers.proto +++ b/python/google/protobuf/internal/test_bad_identifiers.proto @@ -43,10 +43,10 @@ message TestBadIdentifiers { // Make sure these reasonable extension names don't conflict with internal // variables. extend TestBadIdentifiers { - optional string message = 100 [default="foo"]; - optional string descriptor = 101 [default="bar"]; - optional string reflection = 102 [default="baz"]; - optional string service = 103 [default="qux"]; + optional string message = 100 [default = "foo"]; + optional string descriptor = 101 [default = "bar"]; + optional string reflection = 102 [default = "baz"]; + optional string service = 103 [default = "qux"]; } message AnotherMessage {} diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 15f3a19fbd..cdab1c6277 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -798,6 +798,39 @@ class OnlyWorksWithProto2RightNowTests(TextFormatBase): self.RemoveRedundantZeros(text_format.MessageToString(message)), 'text_format_unittest_data_oneof_implemented.txt') + def testPrintUnknownFields(self): + message = unittest_pb2.TestAllTypes() + message.optional_int32 = 101 + message.optional_double = 102.0 + message.optional_string = u'hello' + message.optional_bytes = b'103' + message.optionalgroup.a = 104 + message.optional_nested_message.bb = 105 + all_data = message.SerializeToString() + empty_message = unittest_pb2.TestEmptyMessage() + empty_message.ParseFromString(all_data) + self.assertEqual('1: 101\n' + '12: 4636878028842991616\n' + '14: "hello"\n' + '15: "103"\n' + '16 {\n' + ' 17: 104\n' + '}\n' + '18 {\n' + ' 1: 105\n' + '}\n', + text_format.MessageToString(empty_message, + print_unknown_fields=True)) + self.assertEqual('1: 101 ' + '12: 4636878028842991616 ' + '14: "hello" ' + '15: "103" ' + '16 { 17: 104 } ' + '18 { 1: 105 }', + text_format.MessageToString(empty_message, + print_unknown_fields=True, + as_one_line=True)) + def testPrintInIndexOrder(self): message = unittest_pb2.TestFieldOrderings() # Fields are listed in index order instead of field number. diff --git a/python/google/protobuf/internal/well_known_types.py b/python/google/protobuf/internal/well_known_types.py index 6b4df511e0..4d1166232f 100644 --- a/python/google/protobuf/internal/well_known_types.py +++ b/python/google/protobuf/internal/well_known_types.py @@ -41,11 +41,17 @@ This files defines well known classes which need extra maintenance including: __author__ = 'jieluo@google.com (Jie Luo)' import calendar -import collections from datetime import datetime from datetime import timedelta import six +try: + # Since python 3 + import collections.abc as collections_abc +except ImportError: + # Won't work after python 3.8 + import collections as collections_abc + from google.protobuf.descriptor import FieldDescriptor _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' @@ -88,6 +94,9 @@ class Any(object): return '/' in self.type_url and self.TypeName() == descriptor.full_name +_EPOCH_DATETIME = datetime.utcfromtimestamp(0) + + class Timestamp(object): """Class for Timestamp message type.""" @@ -221,8 +230,9 @@ class Timestamp(object): def ToDatetime(self): """Converts Timestamp to datetime.""" - return datetime.utcfromtimestamp( - self.seconds + self.nanos / float(_NANOS_PER_SECOND)) + return _EPOCH_DATETIME + timedelta( + seconds=self.seconds, microseconds=_RoundTowardZero( + self.nanos, _NANOS_PER_MICROSECOND)) def FromDatetime(self, dt): """Converts datetime to Timestamp.""" @@ -780,7 +790,7 @@ class Struct(object): for key, value in dictionary.items(): _SetStructValue(self.fields[key], value) -collections.MutableMapping.register(Struct) +collections_abc.MutableMapping.register(Struct) class ListValue(object): @@ -824,7 +834,7 @@ class ListValue(object): list_value.Clear() return list_value -collections.MutableSequence.register(ListValue) +collections_abc.MutableSequence.register(ListValue) WKTBASES = { diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 95a7023cf0..61b41ec523 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -34,9 +34,15 @@ __author__ = 'jieluo@google.com (Jie Luo)' -import collections import datetime +try: + # Since python 3 + import collections.abc as collections_abc +except ImportError: + # Won't work after python 3.8 + import collections as collections_abc + try: import unittest2 as unittest #PY26 except ImportError: @@ -249,6 +255,14 @@ class TimeUtilTest(TimeUtilTestBase): self.assertEqual(datetime.datetime(1970, 1, 1, 0, 0, 1, 999000), message.ToDatetime()) + dt = datetime.datetime(2555, 2, 22, 1, 2, 3, 456789) + message.FromDatetime(dt) + self.assertEqual(dt, message.ToDatetime()) + + dt = datetime.datetime.max + message.FromDatetime(dt) + self.assertEqual(dt, message.ToDatetime()) + def testDatetimeConversionWithTimezone(self): class TZ(datetime.tzinfo): @@ -740,7 +754,7 @@ class StructTest(unittest.TestCase): def testStruct(self): struct = struct_pb2.Struct() - self.assertIsInstance(struct, collections.Mapping) + self.assertIsInstance(struct, collections_abc.Mapping) self.assertEqual(0, len(struct)) struct_class = struct.__class__ @@ -749,7 +763,7 @@ class StructTest(unittest.TestCase): struct['key3'] = True struct.get_or_create_struct('key4')['subkey'] = 11.0 struct_list = struct.get_or_create_list('key5') - self.assertIsInstance(struct_list, collections.Sequence) + self.assertIsInstance(struct_list, collections_abc.Sequence) struct_list.extend([6, 'seven', True, False, None]) struct_list.add_struct()['subkey2'] = 9 struct['key6'] = {'subkey': {}} diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index d5813d8004..fb55511ca2 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -249,6 +249,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { } ScopedPyObjectPtr value( PyEval_CallObject(message_class->AsPyObject(), NULL)); + Py_DECREF(message_class); if (value == NULL) { return NULL; } @@ -363,7 +364,7 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, return it->second; } // Create a new descriptor object - PyBaseDescriptor* py_descriptor = PyObject_New( + PyBaseDescriptor* py_descriptor = PyObject_GC_New( PyBaseDescriptor, type); if (py_descriptor == NULL) { return NULL; @@ -385,6 +386,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, Py_INCREF(pool); py_descriptor->pool = pool; + PyObject_GC_Track(py_descriptor); + if (was_created) { *was_created = true; } @@ -398,41 +401,53 @@ static void Dealloc(PyBaseDescriptor* self) { Py_TYPE(self)->tp_free(reinterpret_cast(self)); } +static int GcTraverse(PyObject* pself, visitproc visit, void* arg) { + PyBaseDescriptor* self = reinterpret_cast(pself); + Py_VISIT(self->pool); + return 0; +} + +static int GcClear(PyObject* pself) { + PyBaseDescriptor* self = reinterpret_cast(pself); + Py_CLEAR(self->pool); + return 0; +} + static PyGetSetDef Getters[] = { {NULL} }; PyTypeObject PyBaseDescriptor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".DescriptorBase", // tp_name - sizeof(PyBaseDescriptor), // tp_basicsize - 0, // tp_itemsize - (destructor)Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "Descriptors base class", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - 0, // tp_methods - 0, // tp_members - Getters, // tp_getset + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".DescriptorBase", // tp_name + sizeof(PyBaseDescriptor), // tp_basicsize + 0, // tp_itemsize + (destructor)Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags + "Descriptors base class", // tp_doc + GcTraverse, // tp_traverse + GcClear, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + Getters, // tp_getset }; } // namespace descriptor @@ -1436,45 +1451,45 @@ static PyMethodDef Methods[] = { } // namespace file_descriptor PyTypeObject PyFileDescriptor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".FileDescriptor", // tp_name - sizeof(PyFileDescriptor), // tp_basicsize - 0, // tp_itemsize - (destructor)file_descriptor::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A File Descriptor", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - file_descriptor::Methods, // tp_methods - 0, // tp_members - file_descriptor::Getters, // tp_getset - &descriptor::PyBaseDescriptor_Type, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - 0, // tp_new - PyObject_Del, // tp_free + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".FileDescriptor", // tp_name + sizeof(PyFileDescriptor), // tp_basicsize + 0, // tp_itemsize + (destructor)file_descriptor::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A File Descriptor", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + file_descriptor::Methods, // tp_methods + 0, // tp_members + file_descriptor::Getters, // tp_getset + &descriptor::PyBaseDescriptor_Type, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + 0, // tp_new + PyObject_GC_Del, // tp_free }; PyObject* PyFileDescriptor_FromDescriptor( diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index d0038b10fb..50b290db0a 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -70,7 +70,7 @@ namespace cdescriptor_pool { // Create a Python DescriptorPool object, but does not fill the "pool" // attribute. static PyDescriptorPool* _CreateDescriptorPool() { - PyDescriptorPool* cpool = PyObject_New( + PyDescriptorPool* cpool = PyObject_GC_New( PyDescriptorPool, &PyDescriptorPool_Type); if (cpool == NULL) { return NULL; @@ -88,6 +88,8 @@ static PyDescriptorPool* _CreateDescriptorPool() { return NULL; } + PyObject_GC_Track(cpool); + return cpool; } @@ -165,7 +167,19 @@ static void Dealloc(PyObject* pself) { delete self->descriptor_options; delete self->database; delete self->pool; - Py_TYPE(self)->tp_free(reinterpret_cast(self)); + Py_TYPE(self)->tp_free(pself); +} + +static int GcTraverse(PyObject* pself, visitproc visit, void* arg) { + PyDescriptorPool* self = reinterpret_cast(pself); + Py_VISIT(self->py_message_factory); + return 0; +} + +static int GcClear(PyObject* pself) { + PyDescriptorPool* self = reinterpret_cast(pself); + Py_CLEAR(self->py_message_factory); + return 0; } static PyObject* FindMessageByName(PyObject* self, PyObject* arg) { @@ -629,45 +643,45 @@ static PyMethodDef Methods[] = { } // namespace cdescriptor_pool PyTypeObject PyDescriptorPool_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".DescriptorPool", // tp_name - sizeof(PyDescriptorPool), // tp_basicsize - 0, // tp_itemsize - cdescriptor_pool::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A Descriptor Pool", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - cdescriptor_pool::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - cdescriptor_pool::New, // tp_new - PyObject_Del, // tp_free + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".DescriptorPool", // tp_name + sizeof(PyDescriptorPool), // tp_basicsize + 0, // tp_itemsize + cdescriptor_pool::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags + "A Descriptor Pool", // tp_doc + cdescriptor_pool::GcTraverse, // tp_traverse + cdescriptor_pool::GcClear, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + cdescriptor_pool::Methods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + cdescriptor_pool::New, // tp_new + PyObject_GC_Del, // tp_free }; // This is the DescriptorPool which contains all the definitions from the diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index 8289daeaa7..7ce7513860 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -50,8 +50,6 @@ struct CMessageClass; // // There is normally one pool per process. We make it a Python object only // because it contains many Python references. -// TODO(amauryfa): See whether such objects can appear in reference cycles, and -// consider adding support for the cyclic GC. // // "Methods" that interacts with this DescriptorPool are in the cdescriptor_pool // namespace. diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index b4dba6e918..9a3caa41e3 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -310,12 +310,25 @@ static PyObject* New(PyTypeObject* type, return result.release(); } -static void Dealloc(CMessageClass *self) { +static void Dealloc(PyObject* pself) { + CMessageClass* self = reinterpret_cast(pself); Py_XDECREF(self->py_message_descriptor); Py_XDECREF(self->py_message_factory); - Py_TYPE(self)->tp_free(reinterpret_cast(self)); + return PyType_Type.tp_dealloc(pself); } +static int GcTraverse(PyObject* pself, visitproc visit, void* arg) { + CMessageClass* self = reinterpret_cast(pself); + Py_VISIT(self->py_message_descriptor); + Py_VISIT(self->py_message_factory); + return PyType_Type.tp_traverse(pself, visit, arg); +} + +static int GcClear(PyObject* pself) { + // It's important to keep the descriptor and factory alive, until the + // C++ message is fully destructed. + return PyType_Type.tp_clear(pself); +} // This function inserts and empty weakref at the end of the list of // subclasses for the main protocol buffer Message class. @@ -328,11 +341,17 @@ static int InsertEmptyWeakref(PyTypeObject *base_type) { // hack addresses. For further background and the fix please see // https://bugs.python.org/issue17936. return 0; +#else +#ifdef Py_DEBUG + // The code below causes all new subclasses to append an entry, which is never + // cleared. This is a small memory leak, which we disable in Py_DEBUG mode + // to have stable refcounting checks. #else PyObject *subclasses = base_type->tp_subclasses; if (subclasses && PyList_CheckExact(subclasses)) { return PyList_Append(subclasses, kEmptyWeakref); } +#endif // !Py_DEBUG return 0; #endif // PY_MAJOR_VERSION >= 3 } @@ -451,44 +470,44 @@ static PyObject* GetAttr(CMessageClass* self, PyObject* name) { } // namespace message_meta static PyTypeObject _CMessageClass_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".MessageMeta", // tp_name - sizeof(CMessageClass), // tp_basicsize - 0, // tp_itemsize - (destructor)message_meta::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - (getattrofunc)message_meta::GetAttr, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags - "The metaclass of ProtocolMessages", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - 0, // tp_methods - 0, // tp_members - message_meta::Getters, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - message_meta::New, // tp_new + PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME + ".MessageMeta", // tp_name + sizeof(CMessageClass), // tp_basicsize + 0, // tp_itemsize + message_meta::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + (getattrofunc)message_meta::GetAttr, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, // tp_flags + "The metaclass of ProtocolMessages", // tp_doc + message_meta::GcTraverse, // tp_traverse + message_meta::GcClear, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + 0, // tp_methods + 0, // tp_members + message_meta::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + message_meta::New, // tp_new }; PyTypeObject* CMessageClass_Type = &_CMessageClass_Type; diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 1b0effae73..64aafaf05a 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -129,12 +129,13 @@ struct CMessageClass { const Descriptor* message_descriptor; // Owned reference, used to keep the pointer above alive. + // This reference must stay alive until all message pointers are destructed. PyObject* py_message_descriptor; // The Python MessageFactory used to create the class. It is needed to resolve // fields descriptors, including extensions fields; its C++ MessageFactory is // used to instantiate submessages. - // We own the reference, because it's important to keep the factory alive. + // This reference must stay alive until all message pointers are destructed. PyMessageFactory* py_message_factory; PyObject* AsPyObject() { diff --git a/python/google/protobuf/pyext/message_factory.cc b/python/google/protobuf/pyext/message_factory.cc index a5ff3184cf..5fed13b943 100644 --- a/python/google/protobuf/pyext/message_factory.cc +++ b/python/google/protobuf/pyext/message_factory.cc @@ -69,9 +69,7 @@ PyMessageFactory* NewMessageFactory(PyTypeObject* type, PyDescriptorPool* pool) factory->message_factory = message_factory; factory->pool = pool; - // TODO(amauryfa): When the MessageFactory is not created from the - // DescriptorPool this reference should be owned, not borrowed. - // Py_INCREF(pool); + Py_INCREF(pool); factory->classes_by_descriptor = new PyMessageFactory::ClassesByMessageMap(); @@ -107,19 +105,37 @@ PyObject* New(PyTypeObject* type, PyObject* args, PyObject* kwargs) { static void Dealloc(PyObject* pself) { PyMessageFactory* self = reinterpret_cast(pself); - // TODO(amauryfa): When the MessageFactory is not created from the - // DescriptorPool this reference should be owned, not borrowed. - // Py_CLEAR(self->pool); typedef PyMessageFactory::ClassesByMessageMap::iterator iterator; for (iterator it = self->classes_by_descriptor->begin(); it != self->classes_by_descriptor->end(); ++it) { - Py_DECREF(it->second); + Py_CLEAR(it->second); } delete self->classes_by_descriptor; delete self->message_factory; + Py_CLEAR(self->pool); Py_TYPE(self)->tp_free(pself); } +static int GcTraverse(PyObject* pself, visitproc visit, void* arg) { + PyMessageFactory* self = reinterpret_cast(pself); + Py_VISIT(self->pool); + for (const auto& desc_and_class : *self->classes_by_descriptor) { + Py_VISIT(desc_and_class.second); + } + return 0; +} + +static int GcClear(PyObject* pself) { + PyMessageFactory* self = reinterpret_cast(pself); + // Here it's important to not clear self->pool, so that the C++ DescriptorPool + // is still alive when self->message_factory is destructed. + for (auto& desc_and_class : *self->classes_by_descriptor) { + Py_CLEAR(desc_and_class.second); + } + + return 0; +} + // Add a message class to our database. int RegisterMessageClass(PyMessageFactory* self, const Descriptor* message_descriptor, @@ -234,44 +250,44 @@ static PyGetSetDef Getters[] = { PyTypeObject PyMessageFactory_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) FULL_MODULE_NAME - ".MessageFactory", // tp_name - sizeof(PyMessageFactory), // tp_basicsize - 0, // tp_itemsize - message_factory::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags - "A static Message Factory", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - message_factory::Methods, // tp_methods - 0, // tp_members - message_factory::Getters, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - 0, // tp_alloc - message_factory::New, // tp_new - PyObject_Del, // tp_free + ".MessageFactory", // tp_name + sizeof(PyMessageFactory), // tp_basicsize + 0, // tp_itemsize + message_factory::Dealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, // tp_flags + "A static Message Factory", // tp_doc + message_factory::GcTraverse, // tp_traverse + message_factory::GcClear, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + message_factory::Methods, // tp_methods + 0, // tp_members + message_factory::Getters, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init + 0, // tp_alloc + message_factory::New, // tp_new + PyObject_GC_Del, // tp_free }; bool InitMessageFactory() { diff --git a/python/google/protobuf/pyext/message_factory.h b/python/google/protobuf/pyext/message_factory.h index 06444b0a2a..515c29cdb8 100644 --- a/python/google/protobuf/pyext/message_factory.h +++ b/python/google/protobuf/pyext/message_factory.h @@ -57,9 +57,8 @@ struct PyMessageFactory { // The C++ one creates messages, when the Python one creates classes. MessageFactory* message_factory; - // borrowed reference to a Python DescriptorPool. - // TODO(amauryfa): invert the dependency: the MessageFactory owns the - // DescriptorPool, not the opposite. + // Owned reference to a Python DescriptorPool. + // This reference must stay until the message_factory is destructed. PyDescriptorPool* pool; // Make our own mapping to retrieve Python classes from C++ descriptors. diff --git a/python/google/protobuf/pyext/proto2_api_test.proto b/python/google/protobuf/pyext/proto2_api_test.proto deleted file mode 100644 index 1fd78e8402..0000000000 --- a/python/google/protobuf/pyext/proto2_api_test.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package google.protobuf.python.internal; - -import "google/protobuf/internal/cpp/proto1_api_test.proto"; - -message TestNestedProto1APIMessage { - optional int32 a = 1; - optional TestMessage.NestedMessage b = 2; -} diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index f0b6e5de5f..d8088a1eb7 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -233,13 +233,12 @@ static PyObject* AddMessage(RepeatedCompositeContainer* self, PyObject* value) { static PyObject* AppendMethod(PyObject* pself, PyObject* value) { RepeatedCompositeContainer* self = reinterpret_cast(pself); - PyObject* py_cmsg = AddMessage(self, value); + ScopedPyObjectPtr py_cmsg(AddMessage(self, value)); if (py_cmsg == nullptr) { return nullptr; } - if (PyList_Append(self->child_messages, py_cmsg) < 0) { - Py_DECREF(py_cmsg); + if (PyList_Append(self->child_messages, py_cmsg.get()) < 0) { return nullptr; } @@ -258,7 +257,7 @@ static PyObject* Insert(PyObject* pself, PyObject* args) { return nullptr; } - PyObject* py_cmsg = AddMessage(self, value); + ScopedPyObjectPtr py_cmsg(AddMessage(self, value)); if (py_cmsg == nullptr) { return nullptr; } @@ -277,7 +276,7 @@ static PyObject* Insert(PyObject* pself, PyObject* args) { } } - if (PyList_Insert(self->child_messages, index, py_cmsg) < 0) { + if (PyList_Insert(self->child_messages, index, py_cmsg.get()) < 0) { return nullptr; } Py_RETURN_NONE; diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 1d965fa9bd..557c7e2852 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -51,6 +51,7 @@ if six.PY3: long = int # pylint: disable=redefined-builtin,invalid-name # pylint: disable=g-import-not-at-top +from google.protobuf.internal import decoder from google.protobuf.internal import type_checkers from google.protobuf import descriptor from google.protobuf import text_encoding @@ -128,7 +129,8 @@ def MessageToString(message, use_field_number=False, descriptor_pool=None, indent=0, - message_formatter=None): + message_formatter=None, + print_unknown_fields=False): # type: (...) -> str """Convert protobuf message to text format. @@ -159,6 +161,7 @@ def MessageToString(message, message_formatter: A function(message, indent, as_one_line): unicode|None to custom format selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. Returns: A string of the text formatted protocol buffer message. @@ -167,7 +170,8 @@ def MessageToString(message, printer = _Printer(out, indent, as_utf8, as_one_line, use_short_repeated_primitives, pointy_brackets, use_index_order, float_format, use_field_number, - descriptor_pool, message_formatter) + descriptor_pool, message_formatter, + print_unknown_fields=print_unknown_fields) printer.PrintMessage(message) result = out.getvalue() out.close() @@ -203,11 +207,19 @@ def PrintMessage(message, float_format=None, use_field_number=False, descriptor_pool=None, - message_formatter=None): - printer = _Printer(out, indent, as_utf8, as_one_line, - use_short_repeated_primitives, pointy_brackets, - use_index_order, float_format, use_field_number, - descriptor_pool, message_formatter) + message_formatter=None, + print_unknown_fields=False): + printer = _Printer( + out=out, indent=indent, as_utf8=as_utf8, + as_one_line=as_one_line, + use_short_repeated_primitives=use_short_repeated_primitives, + pointy_brackets=pointy_brackets, + use_index_order=use_index_order, + float_format=float_format, + use_field_number=use_field_number, + descriptor_pool=descriptor_pool, + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields) printer.PrintMessage(message) @@ -221,12 +233,14 @@ def PrintField(field, pointy_brackets=False, use_index_order=False, float_format=None, - message_formatter=None): + message_formatter=None, + print_unknown_fields=False): """Print a single field name/value pair.""" printer = _Printer(out, indent, as_utf8, as_one_line, use_short_repeated_primitives, pointy_brackets, use_index_order, float_format, - message_formatter=message_formatter) + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields) printer.PrintField(field, value) @@ -240,12 +254,14 @@ def PrintFieldValue(field, pointy_brackets=False, use_index_order=False, float_format=None, - message_formatter=None): + message_formatter=None, + print_unknown_fields=False): """Print a single field value (not including name).""" printer = _Printer(out, indent, as_utf8, as_one_line, use_short_repeated_primitives, pointy_brackets, use_index_order, float_format, - message_formatter=message_formatter) + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields) printer.PrintFieldValue(field, value) @@ -274,6 +290,11 @@ def _BuildMessageFromTypeName(type_name, descriptor_pool): return message_type() +# These values must match WireType enum in google/protobuf/wire_format.h. +WIRETYPE_LENGTH_DELIMITED = 2 +WIRETYPE_START_GROUP = 3 + + class _Printer(object): """Text format printer for protocol message.""" @@ -288,7 +309,8 @@ class _Printer(object): float_format=None, use_field_number=False, descriptor_pool=None, - message_formatter=None): + message_formatter=None, + print_unknown_fields=False): """Initialize the Printer. Floating point values can be formatted compactly with 15 digits of @@ -317,6 +339,7 @@ class _Printer(object): message_formatter: A function(message, indent, as_one_line): unicode|None to custom format selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. """ self.out = out self.indent = indent @@ -329,6 +352,7 @@ class _Printer(object): self.use_field_number = use_field_number self.descriptor_pool = descriptor_pool self.message_formatter = message_formatter + self.print_unknown_fields = print_unknown_fields def _TryPrintAsAnyMessage(self, message): """Serializes if message is a google.protobuf.Any field.""" @@ -392,6 +416,64 @@ class _Printer(object): else: self.PrintField(field, value) + if self.print_unknown_fields: + self._PrintUnknownFields(message.UnknownFields()) + + def _PrintUnknownFields(self, unknown_fields): + """Print unknown fields.""" + out = self.out + for field in unknown_fields: + out.write(' ' * self.indent) + out.write(str(field.field_number)) + if field.wire_type == WIRETYPE_START_GROUP: + if self.as_one_line: + out.write(' { ') + else: + out.write(' {\n') + self.indent += 2 + + self._PrintUnknownFields(field.data) + + if self.as_one_line: + out.write('} ') + else: + out.write('}\n') + self.indent -= 2 + elif field.wire_type == WIRETYPE_LENGTH_DELIMITED: + try: + # If this field is parseable as a Message, it is probably + # an embedded message. + # pylint: disable=protected-access + (embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet( + memoryview(field.data), 0, len(field.data)) + except Exception: # pylint: disable=broad-except + pos = 0 + + if pos == len(field.data): + if self.as_one_line: + out.write(' { ') + else: + out.write(' {\n') + self.indent += 2 + + self._PrintUnknownFields(embedded_unknown_message) + + if self.as_one_line: + out.write('} ') + else: + out.write('}\n') + self.indent -= 2 + else: + # A string or bytes field. self.as_utf8 may not work. + out.write(': \"') + out.write(text_encoding.CEscape(field.data, False)) + out.write('\" ' if self.as_one_line else '\"\n') + else: + # varint, fixed32, fixed64 + out.write(': ') + out.write(str(field.data)) + out.write(' ' if self.as_one_line else '\n') + def _PrintFieldName(self, field): """Print field name.""" out = self.out diff --git a/src/Makefile.am b/src/Makefile.am index 169d92352c..6f64b96e27 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -131,7 +131,6 @@ nobase_include_HEADERS = \ google/protobuf/unknown_field_set.h \ google/protobuf/wire_format.h \ google/protobuf/wire_format_lite.h \ - google/protobuf/wire_format_lite_inl.h \ google/protobuf/wrappers.pb.h \ google/protobuf/io/coded_stream.h \ $(GZHEADERS) \ diff --git a/src/google/protobuf/compiler/cpp/cpp_enum.cc b/src/google/protobuf/compiler/cpp/cpp_enum.cc index b728470182..9220ddb1c5 100644 --- a/src/google/protobuf/compiler/cpp/cpp_enum.cc +++ b/src/google/protobuf/compiler/cpp/cpp_enum.cc @@ -72,6 +72,7 @@ EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor, variables_["short_name"] = descriptor_->name(); variables_["enumbase"] = options_.proto_h ? " : int" : ""; variables_["nested_name"] = descriptor_->name(); + variables_["resolved_name"] = ResolveKeyword(descriptor_->name()); variables_["prefix"] = (descriptor_->containing_type() == NULL) ? "" : classname_ + "_"; } @@ -192,13 +193,13 @@ void EnumGenerator::GenerateGetEnumDescriptorSpecializations( void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { Formatter format(printer, variables_); - format("typedef $classname$ $nested_name$;\n"); + format("typedef $classname$ $resolved_name$;\n"); for (int j = 0; j < descriptor_->value_count(); j++) { std::string deprecated_attr = DeprecatedAttribute( options_, descriptor_->value(j)->options().deprecated()); format( - "$1$static constexpr $nested_name$ ${2$$3$$}$ =\n" + "$1$static constexpr $resolved_name$ ${2$$3$$}$ =\n" " $classname$_$3$;\n", deprecated_attr, descriptor_->value(j), EnumValueName(descriptor_->value(j))); @@ -208,9 +209,9 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { "static inline bool $nested_name$_IsValid(int value) {\n" " return $classname$_IsValid(value);\n" "}\n" - "static constexpr $nested_name$ ${1$$nested_name$_MIN$}$ =\n" + "static constexpr $resolved_name$ ${1$$nested_name$_MIN$}$ =\n" " $classname$_$nested_name$_MIN;\n" - "static constexpr $nested_name$ ${1$$nested_name$_MAX$}$ =\n" + "static constexpr $resolved_name$ ${1$$nested_name$_MAX$}$ =\n" " $classname$_$nested_name$_MAX;\n", descriptor_); if (generate_array_size_) { @@ -231,7 +232,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { // version below. Would this break our compatibility guarantees? format( "static inline const std::string& " - "$nested_name$_Name($nested_name$ value) {" + "$nested_name$_Name($resolved_name$ value) {" "\n" " return $classname$_Name(value);\n" "}\n"); @@ -243,7 +244,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { "template\n" "static inline const std::string& $nested_name$_Name(T enum_t_value) " "{\n" - " static_assert(::std::is_same::value ||\n" + " static_assert(::std::is_same::value ||\n" " ::std::is_integral::value,\n" " \"Incorrect type passed to function $nested_name$_Name.\");\n" " return $classname$_Name(enum_t_value);\n" @@ -251,7 +252,7 @@ void EnumGenerator::GenerateSymbolImports(io::Printer* printer) const { } format( "static inline bool $nested_name$_Parse(const std::string& name,\n" - " $nested_name$* value) {\n" + " $resolved_name$* value) {\n" " return $classname$_Parse(name, value);\n" "}\n"); } diff --git a/src/google/protobuf/compiler/cpp/cpp_extension.cc b/src/google/protobuf/compiler/cpp/cpp_extension.cc index 6bcbb3a2b8..b03ca837bc 100644 --- a/src/google/protobuf/compiler/cpp/cpp_extension.cc +++ b/src/google/protobuf/compiler/cpp/cpp_extension.cc @@ -93,7 +93,7 @@ ExtensionGenerator::ExtensionGenerator(const FieldDescriptor* descriptor, variables_["extendee"] = ExtendeeClassName(descriptor_); variables_["type_traits"] = type_traits_; std::string name = descriptor_->name(); - variables_["name"] = name; + variables_["name"] = ResolveKeyword(name); variables_["constant_name"] = FieldConstantName(descriptor_); variables_["field_type"] = StrCat(static_cast(descriptor_->type())); @@ -102,7 +102,7 @@ ExtensionGenerator::ExtensionGenerator(const FieldDescriptor* descriptor, std::string scope = IsScoped() ? ClassName(descriptor_->extension_scope(), false) + "::" : ""; variables_["scope"] = scope; - std::string scoped_name = scope + name; + std::string scoped_name = scope + ResolveKeyword(name); variables_["scoped_name"] = scoped_name; variables_["number"] = StrCat(descriptor_->number()); } diff --git a/src/google/protobuf/compiler/cpp/cpp_helpers.cc b/src/google/protobuf/compiler/cpp/cpp_helpers.cc index 4e19fdcd01..21af61f11d 100644 --- a/src/google/protobuf/compiler/cpp/cpp_helpers.cc +++ b/src/google/protobuf/compiler/cpp/cpp_helpers.cc @@ -319,12 +319,12 @@ std::string ClassName(const Descriptor* descriptor) { if (parent) res += ClassName(parent) + "_"; res += descriptor->name(); if (IsMapEntryMessage(descriptor)) res += "_DoNotUse"; - return res; + return ResolveKeyword(res); } std::string ClassName(const EnumDescriptor* enum_descriptor) { if (enum_descriptor->containing_type() == nullptr) { - return enum_descriptor->name(); + return ResolveKeyword(enum_descriptor->name()); } else { return ClassName(enum_descriptor->containing_type()) + "_" + enum_descriptor->name(); @@ -395,6 +395,13 @@ std::string SuperClassName(const Descriptor* descriptor, : "::MessageLite"); } +std::string ResolveKeyword(const string& name) { + if (kKeywords.count(name) > 0) { + return name + "_"; + } + return name; +} + std::string FieldName(const FieldDescriptor* field) { std::string result = field->name(); LowerString(&result); @@ -1345,8 +1352,6 @@ class ParseLoopGenerator { format_.Set("p_ns", "::" + ProtobufNamespace(options_)); format_.Set("pi_ns", StrCat("::", ProtobufNamespace(options_), "::internal")); format_.Set("GOOGLE_PROTOBUF", MacroPrefix(options_)); - format_.Set("kSlopBytes", - static_cast(internal::ParseContext::kSlopBytes)); std::map vars; SetCommonVars(options_, &vars); format_.AddMap(vars); @@ -1365,93 +1370,14 @@ class ParseLoopGenerator { format_( "const char* $classname$::_InternalParse(const char* ptr, " - "$pi_ns$::ParseContext* ctx) {\n" - " $p_ns$::Arena* arena = GetArena(); (void)arena;\n" - " while (!ctx->Done(&ptr)) {\n" - " $uint32$ tag;\n" - " ptr = $pi_ns$::ReadTag(ptr, &tag);\n" - " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n" - " switch (tag >> 3) {\n"); - + "$pi_ns$::ParseContext* ctx) {\n"); format_.Indent(); - format_.Indent(); - format_.Indent(); - - for (const auto* field : ordered_fields) { - // Print the field's (or oneof's) proto-syntax definition as a comment. - // We don't want to print group bodies so we cut off after the first - // line. - std::string def; - { - DebugStringOptions options; - options.elide_group_body = true; - options.elide_oneof_body = true; - def = field->DebugStringWithOptions(options); - def = def.substr(0, def.find_first_of('\n')); - } - format_( - "// $1$\n" - "case $2$: {\n", - def, field->number()); - format_.Indent(); - GenerateCaseBody(field); - format_.Outdent(); - format_("}\n"); // case - } // for fields - - // Default case - format_("default: {\n"); - if (!ordered_fields.empty()) format_("handle_unusual:\n"); - format_( - " if ((tag & 7) == 4 || tag == 0) {\n" - " ctx->SetLastTag(tag);\n" - " return ptr;\n" - " }\n"); - if (IsMapEntryMessage(descriptor)) { - format_(" break;\n"); - } else { - if (descriptor->extension_range_count() > 0) { - format_("if ("); - for (int i = 0; i < descriptor->extension_range_count(); i++) { - const Descriptor::ExtensionRange* range = - descriptor->extension_range(i); - if (i > 0) format_(" ||\n "); - - uint32 start_tag = WireFormatLite::MakeTag( - range->start, static_cast(0)); - uint32 end_tag = WireFormatLite::MakeTag( - range->end, static_cast(0)); - - if (range->end > FieldDescriptor::kMaxNumber) { - format_("($1$u <= tag)", start_tag); - } else { - format_("($1$u <= tag && tag < $2$u)", start_tag, end_tag); - } - } - format_(") {\n"); - format_( - " ptr = _extensions_.ParseField(tag, ptr, \n" - " internal_default_instance(), &_internal_metadata_, " - "ctx);\n" - " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n" - " break;\n" - "}\n"); - } - format_( - " ptr = UnknownFieldParse(tag,\n" - " _internal_metadata_.mutable_unknown_fields(), ptr, ctx);\n" - " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n" - " break;\n"); + if (descriptor->file()->options().cc_enable_arenas()) { + format_("$p_ns$::Arena* arena = GetArenaNoVirtual(); (void)arena;\n"); } - format_("}\n"); // default case - format_.Outdent(); + GenerateParseLoop(descriptor, ordered_fields); format_.Outdent(); - format_.Outdent(); - format_( - " } // switch\n" - " } // while\n" - " return ptr;\n" - "}\n"); + format_("}\n"); } private: @@ -1469,18 +1395,25 @@ class ParseLoopGenerator { field_name.substr(2)); // remove ", " field_name = ", kFieldName"; } - format_("if (arena != nullptr) {\n"); if (HasFieldPresence(field->file())) { - format_(" HasBitSetters::set_has_$1$(this);\n", FieldName(field)); + format_("HasBitSetters::set_has_$1$(this);\n", FieldName(field)); } + string default_string = + field->default_value_string().empty() + ? "::" + ProtobufNamespace(options_) + + "::internal::GetEmptyStringAlreadyInited()" + : QualifiedClassName(field->containing_type(), options_) + + "::" + MakeDefaultName(field) + ".get()"; format_( + "if (arena != nullptr) {\n" " ptr = $pi_ns$::InlineCopyIntoArenaString$1$(&$2$_, ptr, ctx, " " arena$3$);\n" "} else {\n" - " ptr = $pi_ns$::InlineGreedyStringParser$1$($4$_$2$(), ptr, ctx$3$);" + " ptr = " + "$pi_ns$::InlineGreedyStringParser$1$($2$_.MutableNoArenaNoDefault(&$4$" + "), ptr, ctx$3$);" "\n}\n", - utf8, FieldName(field), field_name, - field->is_repeated() ? "add" : "mutable"); + utf8, FieldName(field), field_name, default_string); } void GenerateStrings(const FieldDescriptor* field, bool check_utf8) { @@ -1766,6 +1699,93 @@ class ParseLoopGenerator { GenerateCaseBody(wiretype, field); } } + + void GenerateParseLoop( + const Descriptor* descriptor, + const std::vector& ordered_fields) { + format_( + "while (!ctx->Done(&ptr)) {\n" + " $uint32$ tag;\n" + " ptr = $pi_ns$::ReadTag(ptr, &tag);\n" + " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr);\n" + " switch (tag >> 3) {\n"); + + format_.Indent(); + format_.Indent(); + + for (const auto* field : ordered_fields) { + // Print the field's (or oneof's) proto-syntax definition as a comment. + // We don't want to print group bodies so we cut off after the first + // line. + std::string def; + { + DebugStringOptions options; + options.elide_group_body = true; + options.elide_oneof_body = true; + def = field->DebugStringWithOptions(options); + def = def.substr(0, def.find_first_of('\n')); + } + format_( + "// $1$\n" + "case $2$: {\n", + def, field->number()); + format_.Indent(); + GenerateCaseBody(field); + format_.Outdent(); + format_("}\n"); // case + } // for fields + + // Default case + format_("default: {\n"); + if (!ordered_fields.empty()) format_("handle_unusual:\n"); + format_( + " if ((tag & 7) == 4 || tag == 0) {\n" + " ctx->SetLastTag(tag);\n" + " return ptr;\n" + " }\n"); + if (IsMapEntryMessage(descriptor)) { + format_(" break;\n"); + } else { + if (descriptor->extension_range_count() > 0) { + format_("if ("); + for (int i = 0; i < descriptor->extension_range_count(); i++) { + const Descriptor::ExtensionRange* range = + descriptor->extension_range(i); + if (i > 0) format_(" ||\n "); + + uint32 start_tag = WireFormatLite::MakeTag( + range->start, static_cast(0)); + uint32 end_tag = WireFormatLite::MakeTag( + range->end, static_cast(0)); + + if (range->end > FieldDescriptor::kMaxNumber) { + format_("($1$u <= tag)", start_tag); + } else { + format_("($1$u <= tag && tag < $2$u)", start_tag, end_tag); + } + } + format_(") {\n"); + format_( + " ptr = _extensions_.ParseField(tag, ptr, \n" + " internal_default_instance(), &_internal_metadata_, " + "ctx);\n" + " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n" + " break;\n" + "}\n"); + } + format_( + " ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);\n" + " $GOOGLE_PROTOBUF$_PARSER_ASSERT(ptr != nullptr);\n" + " break;\n"); + } + format_("}\n"); // default case + format_.Outdent(); + format_.Outdent(); + format_( + " } // switch\n" + "} // while\n" + "return ptr;\n"); + } }; void GenerateParserLoop(const Descriptor* descriptor, const Options& options, diff --git a/src/google/protobuf/compiler/cpp/cpp_helpers.h b/src/google/protobuf/compiler/cpp/cpp_helpers.h index 2e4ba4f35e..895749db01 100644 --- a/src/google/protobuf/compiler/cpp/cpp_helpers.h +++ b/src/google/protobuf/compiler/cpp/cpp_helpers.h @@ -143,6 +143,9 @@ std::string ReferenceFunctionName(const Descriptor* descriptor, std::string SuperClassName(const Descriptor* descriptor, const Options& options); +// Adds an underscore if necessary to prevent conflicting with a keyword. +std::string ResolveKeyword(const string& name); + // Get the (unqualified) name that should be used for this field in C++ code. // The name is coerced to lower-case to emulate proto1 behavior. People // should be using lowercase-with-underscores style for proto field names diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 28543980ea..bd17215d0d 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -975,9 +975,11 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { GOOGLE_CHECK(suffix == "UTF8Verify"); format( " bool ValidateKey() const {\n" + "#ifndef NDEBUG\n" " ::$proto_ns$::internal::WireFormatLite::VerifyUtf8String(\n" " key().data(), key().size(), ::$proto_ns$::internal::" "WireFormatLite::PARSE, \"$1$\");\n" + "#endif\n" " return true;\n" " }\n", descriptor_->field(0)->full_name()); @@ -999,9 +1001,11 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { GOOGLE_CHECK(suffix == "UTF8Verify"); format( " bool ValidateValue() const {\n" + "#ifndef NDEBUG\n" " ::$proto_ns$::internal::WireFormatLite::VerifyUtf8String(\n" " value().data(), value().size(), ::$proto_ns$::internal::" "WireFormatLite::PARSE, \"$1$\");\n" + "#endif\n" " return true;\n" " }\n", descriptor_->field(1)->full_name()); @@ -1324,7 +1328,7 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* printer) { const Descriptor* nested_type = descriptor_->nested_type(i); if (!IsMapEntryMessage(nested_type)) { format.Set("nested_full_name", ClassName(nested_type, false)); - format.Set("nested_name", nested_type->name()); + format.Set("nested_name", ResolveKeyword(nested_type->name())); format("typedef ${1$$nested_full_name$$}$ ${1$$nested_name$$}$;\n", nested_type); } diff --git a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto index 0f15f39ac7..479710821f 100644 --- a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto +++ b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto @@ -35,11 +35,13 @@ // This file tests that various identifiers work as field and type names even // though the same identifiers are used internally by the C++ code generator. +// LINT: LEGACY_NAMES + syntax = "proto2"; // Some generic_services option(s) added automatically. // See: http://go/proto2-generic-services-default -option cc_generic_services = true; // auto-added +option cc_generic_services = true; // auto-added // We don't put this in a package within proto2 because we need to make sure // that the generated code doesn't depend on being in the proto2 namespace. @@ -55,17 +57,29 @@ message TestConflictingSymbolNames { optional int32 output = 2; optional string length = 3; repeated int32 i = 4; - repeated string new_element = 5 [ctype=STRING_PIECE]; + repeated string new_element = 5 [ctype = STRING_PIECE]; optional int32 total_size = 6; optional int32 tag = 7; enum TestEnum { FOO = 0; } - message Data1 { repeated int32 data = 1; } - message Data2 { repeated TestEnum data = 1; } - message Data3 { repeated string data = 1; } - message Data4 { repeated Data4 data = 1; } - message Data5 { repeated string data = 1 [ctype=STRING_PIECE]; } - message Data6 { repeated string data = 1 [ctype=CORD]; } + message Data1 { + repeated int32 data = 1; + } + message Data2 { + repeated TestEnum data = 1; + } + message Data3 { + repeated string data = 1; + } + message Data4 { + repeated Data4 data = 1; + } + message Data5 { + repeated string data = 1 [ctype = STRING_PIECE]; + } + message Data6 { + repeated string data = 1 [ctype = CORD]; + } optional int32 source = 8; optional int32 value = 9; @@ -91,10 +105,10 @@ message TestConflictingSymbolNames { optional uint32 reflection = 27; message Cord {} - optional string some_cord = 28 [ctype=CORD]; + optional string some_cord = 28 [ctype = CORD]; message StringPiece {} - optional string some_string_piece = 29 [ctype=STRING_PIECE]; + optional string some_string_piece = 29 [ctype = STRING_PIECE]; // Some keywords. optional uint32 int = 30; @@ -125,14 +139,15 @@ message TestConflictingSymbolNames { extensions 1000 to max; // NO_PROTO3 } -message TestConflictingSymbolNamesExtension { // NO_PROTO3 - extend TestConflictingSymbolNames { // NO_PROTO3 - repeated int32 repeated_int32_ext = 20423638 [packed=true]; // NO_PROTO3 - } // NO_PROTO3 -} // NO_PROTO3 +message TestConflictingSymbolNamesExtension { // NO_PROTO3 + extend TestConflictingSymbolNames { // NO_PROTO3 + repeated int32 repeated_int32_ext = 20423638 [packed = true]; // NO_PROTO3 + } // NO_PROTO3 +} // NO_PROTO3 message TestConflictingEnumNames { // NO_PROTO3 - enum NestedConflictingEnum { // NO_PROTO3 + enum while { // NO_PROTO3 + default = 0; // NO_PROTO3 and = 1; // NO_PROTO3 class = 2; // NO_PROTO3 int = 3; // NO_PROTO3 @@ -140,18 +155,26 @@ message TestConflictingEnumNames { // NO_PROTO3 XOR = 5; // NO_PROTO3 } // NO_PROTO3 - optional NestedConflictingEnum conflicting_enum = 1; // NO_PROTO3 + optional while conflicting_enum = 1; // NO_PROTO3 } // NO_PROTO3 -enum ConflictingEnum { // NO_PROTO3 +enum bool { // NO_PROTO3 + default = 0; // NO_PROTO3 NOT_EQ = 1; // NO_PROTO3 volatile = 2; // NO_PROTO3 return = 3; // NO_PROTO3 - NULL = 4; // NO_PROTO3 } // NO_PROTO3 message DummyMessage {} +message NULL { + optional int32 int = 1; +} + +extend TestConflictingSymbolNames { // NO_PROTO3 + optional int32 void = 314253; // NO_PROTO3 +} // NO_PROTO3 + // Message names that could conflict. message Shutdown {} message TableStruct {} diff --git a/src/google/protobuf/compiler/cpp/cpp_unittest.cc b/src/google/protobuf/compiler/cpp/cpp_unittest.cc index a17ed8bd70..4a52460d3f 100644 --- a/src/google/protobuf/compiler/cpp/cpp_unittest.cc +++ b/src/google/protobuf/compiler/cpp/cpp_unittest.cc @@ -102,20 +102,33 @@ TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingSymbolNames) { TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingEnumNames) { protobuf_unittest::TestConflictingEnumNames message; - message.set_conflicting_enum(protobuf_unittest::TestConflictingEnumNames_NestedConflictingEnum_and_); + message.set_conflicting_enum( + protobuf_unittest::TestConflictingEnumNames_while_and_); EXPECT_EQ(1, message.conflicting_enum()); - message.set_conflicting_enum(protobuf_unittest::TestConflictingEnumNames_NestedConflictingEnum_XOR); + message.set_conflicting_enum( + protobuf_unittest::TestConflictingEnumNames_while_XOR); EXPECT_EQ(5, message.conflicting_enum()); - protobuf_unittest::ConflictingEnum conflicting_enum; + protobuf_unittest::bool_ conflicting_enum; conflicting_enum = protobuf_unittest::NOT_EQ; EXPECT_EQ(1, conflicting_enum); conflicting_enum = protobuf_unittest::return_; EXPECT_EQ(3, conflicting_enum); - conflicting_enum = protobuf_unittest::NULL_; - EXPECT_EQ(4, conflicting_enum); } +TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingMessageNames) { + protobuf_unittest::NULL_ message; + message.set_int_(123); + EXPECT_EQ(message.int_(), 123); +} + +TEST(GENERATED_MESSAGE_TEST_NAME, TestConflictingExtension) { + protobuf_unittest::TestConflictingSymbolNames message; + message.SetExtension(protobuf_unittest::void_, 123); + EXPECT_EQ(123, message.GetExtension(protobuf_unittest::void_)); +} + + } // namespace cpp_unittest } // namespace cpp } // namespace compiler diff --git a/src/google/protobuf/compiler/java/java_enum_field_lite.cc b/src/google/protobuf/compiler/java/java_enum_field_lite.cc index 1f4317d0e1..725997577d 100644 --- a/src/google/protobuf/compiler/java/java_enum_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_enum_field_lite.cc @@ -53,6 +53,13 @@ namespace compiler { namespace java { namespace { +bool EnableExperimentalRuntimeForLite() { +#ifdef PROTOBUF_EXPERIMENT + return PROTOBUF_EXPERIMENT; +#else // PROTOBUF_EXPERIMENT + return false; +#endif // !PROTOBUF_EXPERIMENT +} void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex, int builderBitIndex, const FieldGeneratorInfo* info, @@ -656,7 +663,7 @@ GenerateMembers(io::Printer* printer) const { printer->Annotate("{", "}", descriptor_); } - if (descriptor_->is_packed() && + if (!EnableExperimentalRuntimeForLite() && descriptor_->is_packed() && context_->HasGeneratedMethods(descriptor_->containing_type())) { printer->Print(variables_, "private int $name$MemoizedSerializedSize;\n"); @@ -929,6 +936,8 @@ GenerateParsingDoneCode(io::Printer* printer) const { void RepeatedImmutableEnumFieldLiteGenerator:: GenerateSerializationCode(io::Printer* printer) const { + GOOGLE_CHECK(!EnableExperimentalRuntimeForLite()); + if (descriptor_->is_packed()) { printer->Print(variables_, "if (get$capitalized_name$List().size() > 0) {\n" @@ -948,6 +957,8 @@ GenerateSerializationCode(io::Printer* printer) const { void RepeatedImmutableEnumFieldLiteGenerator:: GenerateSerializedSizeCode(io::Printer* printer) const { + GOOGLE_CHECK(!EnableExperimentalRuntimeForLite()); + printer->Print(variables_, "{\n" " int dataSize = 0;\n"); diff --git a/src/google/protobuf/compiler/java/java_helpers.cc b/src/google/protobuf/compiler/java/java_helpers.cc index 3b397d3fe0..c6e4fea55a 100644 --- a/src/google/protobuf/compiler/java/java_helpers.cc +++ b/src/google/protobuf/compiler/java/java_helpers.cc @@ -105,6 +105,7 @@ std::string FieldName(const FieldDescriptor* field) { return field_name; } + } // namespace void PrintGeneratedAnnotation(io::Printer* printer, char delimiter, diff --git a/src/google/protobuf/compiler/java/java_primitive_field_lite.cc b/src/google/protobuf/compiler/java/java_primitive_field_lite.cc index 0fda38db05..be5f0f1ac5 100644 --- a/src/google/protobuf/compiler/java/java_primitive_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_primitive_field_lite.cc @@ -56,6 +56,13 @@ using internal::WireFormat; using internal::WireFormatLite; namespace { +bool EnableExperimentalRuntimeForLite() { +#ifdef PROTOBUF_EXPERIMENT + return PROTOBUF_EXPERIMENT; +#else // PROTOBUF_EXPERIMENT + return false; +#endif // !PROTOBUF_EXPERIMENT +} void SetPrimitiveVariables(const FieldDescriptor* descriptor, int messageBitIndex, int builderBitIndex, @@ -663,7 +670,7 @@ GenerateMembers(io::Printer* printer) const { "}\n"); printer->Annotate("{", "}", descriptor_); - if (descriptor_->is_packed() && + if (!EnableExperimentalRuntimeForLite() && descriptor_->is_packed() && context_->HasGeneratedMethods(descriptor_->containing_type())) { printer->Print(variables_, "private int $name$MemoizedSerializedSize = -1;\n"); @@ -854,6 +861,8 @@ GenerateParsingDoneCode(io::Printer* printer) const { void RepeatedImmutablePrimitiveFieldLiteGenerator:: GenerateSerializationCode(io::Printer* printer) const { + GOOGLE_CHECK(!EnableExperimentalRuntimeForLite()); + if (descriptor_->is_packed()) { // We invoke getSerializedSize in writeTo for messages that have packed // fields in ImmutableMessageGenerator::GenerateMessageSerializationMethods. @@ -876,6 +885,8 @@ GenerateSerializationCode(io::Printer* printer) const { void RepeatedImmutablePrimitiveFieldLiteGenerator:: GenerateSerializedSizeCode(io::Printer* printer) const { + GOOGLE_CHECK(!EnableExperimentalRuntimeForLite()); + printer->Print(variables_, "{\n" " int dataSize = 0;\n"); diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index f3d8533d78..e00637e6ea 100644 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -2426,9 +2426,13 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options, // all). So we want to generate independent code. // The accessor for unset optional values without default should return // null. Those are converted to undefined in the generated object. - printer->Print("(f = "); + if (!use_default) { + printer->Print("(f = "); + } GenerateFieldValueExpression(printer, "msg", field, use_default); - printer->Print(") == null ? undefined : f"); + if (!use_default) { + printer->Print(") == null ? undefined : f"); + } } } diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 58e5694778..98280f85d9 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -62,8 +62,8 @@ namespace { class MockErrorCollector : public io::ErrorCollector { public: - MockErrorCollector() {} - ~MockErrorCollector() {} + MockErrorCollector() = default; + ~MockErrorCollector() override = default; std::string text_; @@ -1114,17 +1114,17 @@ TEST_F(ParseMiscTest, ParsePackageWithSpaces) { TEST_F(ParseMiscTest, ParseFileOptions) { ExpectParsesTo( - "option java_package = \"com.google.foo\";\n" - "option optimize_for = CODE_SIZE;", - - "options {" - "uninterpreted_option { name { name_part: \"java_package\" " - " is_extension: false }" - " string_value: \"com.google.foo\"} " - "uninterpreted_option { name { name_part: \"optimize_for\" " - " is_extension: false }" - " identifier_value: \"CODE_SIZE\" } " - "}"); + "option java_package = \"com.google.foo\";\n" + "option optimize_for = CODE_SIZE;", + + "options {" + "uninterpreted_option { name { name_part: \"java_package\" " + " is_extension: false }" + " string_value: \"com.google.foo\"} " + "uninterpreted_option { name { name_part: \"optimize_for\" " + " is_extension: false }" + " identifier_value: \"CODE_SIZE\" } " + "}"); } // =================================================================== diff --git a/src/google/protobuf/compiler/plugin.proto b/src/google/protobuf/compiler/plugin.proto index 5b5574529e..665e5a725d 100644 --- a/src/google/protobuf/compiler/plugin.proto +++ b/src/google/protobuf/compiler/plugin.proto @@ -45,6 +45,7 @@ // flag "--${NAME}_out" is passed to protoc. syntax = "proto2"; + package google.protobuf.compiler; option java_package = "com.google.protobuf.compiler"; option java_outer_classname = "PluginProtos"; diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index 795df3ddd6..d0a63b9307 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -279,14 +279,14 @@ void ExtensionSet::ClearExtension(int number) { namespace { enum { - REPEATED, - OPTIONAL + REPEATED_FIELD, + OPTIONAL_FIELD }; } // namespace -#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE) \ - GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED : OPTIONAL, LABEL); \ +#define GOOGLE_DCHECK_TYPE(EXTENSION, LABEL, CPPTYPE) \ + GOOGLE_DCHECK_EQ((EXTENSION).is_repeated ? REPEATED_FIELD : OPTIONAL_FIELD, LABEL); \ GOOGLE_DCHECK_EQ(cpp_type((EXTENSION).type), WireFormatLite::CPPTYPE_##CPPTYPE) // ------------------------------------------------------------------- @@ -300,7 +300,7 @@ LOWERCASE ExtensionSet::Get##CAMELCASE(int number, \ if (extension == NULL || extension->is_cleared) { \ return default_value; \ } else { \ - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE); \ + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE); \ return extension->LOWERCASE##_value; \ } \ } \ @@ -314,7 +314,7 @@ void ExtensionSet::Set##CAMELCASE(int number, FieldType type, \ GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_##UPPERCASE); \ extension->is_repeated = false; \ } else { \ - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, UPPERCASE); \ + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, UPPERCASE); \ } \ extension->is_cleared = false; \ extension->LOWERCASE##_value = value; \ @@ -323,7 +323,7 @@ void ExtensionSet::Set##CAMELCASE(int number, FieldType type, \ LOWERCASE ExtensionSet::GetRepeated##CAMELCASE(int number, int index) const { \ const Extension* extension = FindOrNull(number); \ GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ return extension->repeated_##LOWERCASE##_value->Get(index); \ } \ \ @@ -331,7 +331,7 @@ void ExtensionSet::SetRepeated##CAMELCASE( \ int number, int index, LOWERCASE value) { \ Extension* extension = FindOrNull(number); \ GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ extension->repeated_##LOWERCASE##_value->Set(index, value); \ } \ \ @@ -347,7 +347,7 @@ void ExtensionSet::Add##CAMELCASE(int number, FieldType type, \ extension->repeated_##LOWERCASE##_value = \ Arena::CreateMessage >(arena_); \ } else { \ - GOOGLE_DCHECK_TYPE(*extension, REPEATED, UPPERCASE); \ + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, UPPERCASE); \ GOOGLE_DCHECK_EQ(extension->is_packed, packed); \ } \ extension->repeated_##LOWERCASE##_value->Add(value); \ @@ -456,7 +456,7 @@ int ExtensionSet::GetEnum(int number, int default_value) const { // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM); return extension->enum_value; } } @@ -469,7 +469,7 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value, GOOGLE_DCHECK_EQ(cpp_type(extension->type), WireFormatLite::CPPTYPE_ENUM); extension->is_repeated = false; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, ENUM); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, ENUM); } extension->is_cleared = false; extension->enum_value = value; @@ -478,14 +478,14 @@ void ExtensionSet::SetEnum(int number, FieldType type, int value, int ExtensionSet::GetRepeatedEnum(int number, int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); return extension->repeated_enum_value->Get(index); } void ExtensionSet::SetRepeatedEnum(int number, int index, int value) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); extension->repeated_enum_value->Set(index, value); } @@ -501,7 +501,7 @@ void ExtensionSet::AddEnum(int number, FieldType type, extension->repeated_enum_value = Arena::CreateMessage >(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, ENUM); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, ENUM); GOOGLE_DCHECK_EQ(extension->is_packed, packed); } extension->repeated_enum_value->Add(value); @@ -517,7 +517,7 @@ const std::string& ExtensionSet::GetString( // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING); return *extension->string_value; } } @@ -531,7 +531,7 @@ std::string* ExtensionSet::MutableString(int number, FieldType type, extension->is_repeated = false; extension->string_value = Arena::Create(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, STRING); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, STRING); } extension->is_cleared = false; return extension->string_value; @@ -541,14 +541,14 @@ const std::string& ExtensionSet::GetRepeatedString(int number, int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); return extension->repeated_string_value->Get(index); } std::string* ExtensionSet::MutableRepeatedString(int number, int index) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); return extension->repeated_string_value->Mutable(index); } @@ -563,7 +563,7 @@ std::string* ExtensionSet::AddString(int number, FieldType type, extension->repeated_string_value = Arena::CreateMessage>(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, STRING); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, STRING); } return extension->repeated_string_value->Add(); } @@ -578,7 +578,7 @@ const MessageLite& ExtensionSet::GetMessage( // Not present. Return the default value. return default_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { return extension->lazymessage_value->GetMessage(default_value); } else { @@ -605,7 +605,7 @@ MessageLite* ExtensionSet::MutableMessage(int number, FieldType type, extension->is_cleared = false; return extension->message_value; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); extension->is_cleared = false; if (extension->is_lazy) { return extension->lazymessage_value->MutableMessage(prototype); @@ -644,7 +644,7 @@ void ExtensionSet::SetAllocatedMessage(int number, FieldType type, extension->message_value->CheckTypeAndMergeFrom(*message); } } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { extension->lazymessage_value->SetAllocatedMessage(message); } else { @@ -680,7 +680,7 @@ void ExtensionSet::UnsafeArenaSetAllocatedMessage( extension->is_lazy = false; extension->message_value = message; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); if (extension->is_lazy) { extension->lazymessage_value->UnsafeArenaSetAllocatedMessage(message); } else { @@ -700,7 +700,7 @@ MessageLite* ExtensionSet::ReleaseMessage(int number, // Not present. Return NULL. return NULL; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); MessageLite* ret = NULL; if (extension->is_lazy) { ret = extension->lazymessage_value->ReleaseMessage(prototype); @@ -729,7 +729,7 @@ MessageLite* ExtensionSet::UnsafeArenaReleaseMessage( // Not present. Return NULL. return NULL; } else { - GOOGLE_DCHECK_TYPE(*extension, OPTIONAL, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, OPTIONAL_FIELD, MESSAGE); MessageLite* ret = NULL; if (extension->is_lazy) { ret = extension->lazymessage_value->UnsafeArenaReleaseMessage(prototype); @@ -752,14 +752,14 @@ const MessageLite& ExtensionSet::GetRepeatedMessage( int number, int index) const { const Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); return extension->repeated_message_value->Get(index); } MessageLite* ExtensionSet::MutableRepeatedMessage(int number, int index) { Extension* extension = FindOrNull(number); GOOGLE_CHECK(extension != NULL) << "Index out-of-bounds (field is empty)."; - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); return extension->repeated_message_value->Mutable(index); } @@ -774,7 +774,7 @@ MessageLite* ExtensionSet::AddMessage(int number, FieldType type, extension->repeated_message_value = Arena::CreateMessage >(arena_); } else { - GOOGLE_DCHECK_TYPE(*extension, REPEATED, MESSAGE); + GOOGLE_DCHECK_TYPE(*extension, REPEATED_FIELD, MESSAGE); } // RepeatedPtrField does not know how to Add() since it cannot diff --git a/src/google/protobuf/extension_set_inl.h b/src/google/protobuf/extension_set_inl.h index 4bfcf3edb6..4e6995ef1e 100644 --- a/src/google/protobuf/extension_set_inl.h +++ b/src/google/protobuf/extension_set_inl.h @@ -235,11 +235,13 @@ const char* ExtensionSet::ParseMessageSetItemTmpl(const char* ptr, extension.descriptor); const char* p; + // We can't use regular parse from string as we have to track + // proper recursion depth and descriptor pools. ParseContext tmp_ctx(ctx->depth(), false, &p, payload); tmp_ctx.data().pool = ctx->data().pool; tmp_ctx.data().factory = ctx->data().factory; - GOOGLE_PROTOBUF_PARSER_ASSERT( - tmp_ctx.AtLegitimateEnd(value->_InternalParse(p, &tmp_ctx))); + GOOGLE_PROTOBUF_PARSER_ASSERT(value->_InternalParse(p, &tmp_ctx) && + tmp_ctx.EndedAtLimit()); } type_id = 0; } diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index c0652c3243..443a5db107 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -50,6 +50,8 @@ #include +#include + #define GOOGLE_PROTOBUF_HAS_ONEOF namespace google { diff --git a/src/google/protobuf/generated_message_table_driven.h b/src/google/protobuf/generated_message_table_driven.h index 857545e747..8564f5e30f 100644 --- a/src/google/protobuf/generated_message_table_driven.h +++ b/src/google/protobuf/generated_message_table_driven.h @@ -273,23 +273,24 @@ inline uint8* TableSerializeToArray(const MessageLite& msg, template struct CompareHelper { - bool operator()(const T& a, const T& b) { return a < b; } + bool operator()(const T& a, const T& b) const { return a < b; } }; template <> struct CompareHelper { - bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) { + bool operator()(const ArenaStringPtr& a, const ArenaStringPtr& b) const { return a.Get() < b.Get(); } }; struct CompareMapKey { template - bool operator()(const MapEntryHelper& a, const MapEntryHelper& b) { + bool operator()(const MapEntryHelper& a, + const MapEntryHelper& b) const { return Compare(a.key_, b.key_); } template - bool Compare(const T& a, const T& b) { + bool Compare(const T& a, const T& b) const { return CompareHelper()(a, b); } }; diff --git a/src/google/protobuf/generated_message_util.cc b/src/google/protobuf/generated_message_util.cc index fe91bcb070..26a422f4ec 100644 --- a/src/google/protobuf/generated_message_util.cc +++ b/src/google/protobuf/generated_message_util.cc @@ -35,9 +35,13 @@ #include #include + +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP // We're only using this as a standard way for getting the thread id. // We're not using any thread functionality. #include // NOLINT +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + #include #include @@ -784,8 +788,17 @@ void InitSCCImpl(SCCInfoBase* scc) { static WrappedMutex mu{GOOGLE_PROTOBUF_LINKER_INITIALIZED}; // Either the default in case no initialization is running or the id of the // thread that is currently initializing. +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP static std::atomic runner; auto me = std::this_thread::get_id(); +#else + // This is a lightweight replacement for std::thread::id. std::thread does not + // work on Windows XP SP2 with the latest VC++ libraries, because it utilizes + // the Concurrency Runtime that is only supported on Windows XP SP3 and above. + static std::atomic_llong runner(-1); + auto me = ::GetCurrentThreadId(); +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + // This will only happen because the constructor will call InitSCC while // constructing the default instance. if (runner.load(std::memory_order_relaxed) == me) { @@ -799,7 +812,13 @@ void InitSCCImpl(SCCInfoBase* scc) { mu.Lock(); runner.store(me, std::memory_order_relaxed); InitSCC_DFS(scc); + +#ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP runner.store(std::thread::id{}, std::memory_order_relaxed); +#else + runner.store(-1, std::memory_order_relaxed); +#endif // #ifndef GOOGLE_PROTOBUF_SUPPORT_WINDOWS_XP + mu.Unlock(); } diff --git a/src/google/protobuf/implicit_weak_message.cc b/src/google/protobuf/implicit_weak_message.cc index 4cd09688d0..35e64f0758 100644 --- a/src/google/protobuf/implicit_weak_message.cc +++ b/src/google/protobuf/implicit_weak_message.cc @@ -41,17 +41,18 @@ namespace google { namespace protobuf { namespace internal { -bool ImplicitWeakMessage::MergePartialFromCodedStream(io::CodedInputStream* input) { - io::StringOutputStream string_stream(&data_); - io::CodedOutputStream coded_stream(&string_stream, false); - return WireFormatLite::SkipMessage(input, &coded_stream); -} - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER const char* ImplicitWeakMessage::_InternalParse(const char* ptr, ParseContext* ctx) { return ctx->AppendString(ptr, &data_); } +#else +bool ImplicitWeakMessage::MergePartialFromCodedStream( + io::CodedInputStream* input) { + io::StringOutputStream string_stream(&data_); + io::CodedOutputStream coded_stream(&string_stream, false); + return WireFormatLite::SkipMessage(input, &coded_stream); +} #endif ExplicitlyConstructed diff --git a/src/google/protobuf/implicit_weak_message.h b/src/google/protobuf/implicit_weak_message.h index 16b916439f..cab5df975a 100644 --- a/src/google/protobuf/implicit_weak_message.h +++ b/src/google/protobuf/implicit_weak_message.h @@ -76,10 +76,10 @@ class PROTOBUF_EXPORT ImplicitWeakMessage : public MessageLite { data_.append(static_cast(other).data_); } - bool MergePartialFromCodedStream(io::CodedInputStream* input) override; - #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER const char* _InternalParse(const char* ptr, ParseContext* ctx) final; +#else + bool MergePartialFromCodedStream(io::CodedInputStream* input) override; #endif size_t ByteSizeLong() const override { return data_.size(); } diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 891f6dd28c..2f1f270a17 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -710,6 +710,9 @@ class PROTOBUF_EXPORT CodedOutputStream { // Skips a number of bytes, leaving the bytes unmodified in the underlying // buffer. Returns false if an underlying write error occurs. This is // mainly useful with GetDirectBufferPointer(). + // Note of caution, the skipped bytes may contain uninitialized data. The + // caller must make sure that the skipped bytes are properly initialized, + // otherwise you might leak bytes from your heap. bool Skip(int count); // Sets *data to point directly at the unwritten part of the diff --git a/src/google/protobuf/lite_unittest.cc b/src/google/protobuf/lite_unittest.cc index 008b323fe6..65b2f523ed 100644 --- a/src/google/protobuf/lite_unittest.cc +++ b/src/google/protobuf/lite_unittest.cc @@ -44,6 +44,7 @@ #include #include #include +#include namespace google { namespace protobuf { @@ -1049,5 +1050,54 @@ TEST(Lite, MapCrash) { "\202\1\15\10\1\200\200\200\200\200\200\200\200\200\200\1")); } +TEST(Lite, CorrectEnding) { + protobuf_unittest::TestAllTypesLite msg; + { + // All proto wireformat parsers should act the same on parsing data in as + // much as it concerns the parsing, ie. not the interpretation of the data. + // TestAllTypesLite is not a group inside another message. So in practice + // will not encounter an end-group tag. However the parser should behave + // like any wire format parser should. + static const char kWireFormat[] = "\204\1"; + io::CodedInputStream cis(reinterpret_cast(kWireFormat), 2); + // The old CodedInputStream parser got an optimization (ReadTagNoLastTag) + // for non-group messages (like TestAllTypesLite) which made it not accept + // end-group. This is not a real big deal, but I think going forward its + // good to have all parse loops behave 'exactly' the same. +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + EXPECT_TRUE(msg.MergePartialFromCodedStream(&cis)); + EXPECT_FALSE(cis.ConsumedEntireMessage()); + EXPECT_TRUE(cis.LastTagWas(132)); +#endif + } + { + // This is an incomplete end-group tag. This should be a genuine parse + // failure. + static const char kWireFormat[] = "\214"; + io::CodedInputStream cis(reinterpret_cast(kWireFormat), 1); + // Unfortunately the old parser detects a parse error in ReadTag and returns + // 0 (as it states 0 is an invalid tag). However 0 is not an invalid tag + // as it can be used to terminate the stream, so this returns true. +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + EXPECT_FALSE(msg.MergePartialFromCodedStream(&cis)); +#endif + } +} + +TEST(Lite, DebugString) { + protobuf_unittest::TestAllTypesLite message1, message2; + EXPECT_TRUE(HasPrefixString(message1.DebugString(), "MessageLite at 0x")); + EXPECT_TRUE(HasPrefixString(message2.DebugString(), "MessageLite at 0x")); + + // DebugString() and ShortDebugString() are the same for now. + EXPECT_EQ(message1.DebugString(), message1.ShortDebugString()); + + // Even identical lite protos should have different DebugString() output. Part + // of the reason for including the memory address is so that we get some + // non-determinism, which should make it easier for us to change the output + // later without breaking any code. + EXPECT_NE(message1.DebugString(), message2.DebugString()); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/map_entry_lite.h b/src/google/protobuf/map_entry_lite.h index 19f0b5fcb5..756b9af0ca 100644 --- a/src/google/protobuf/map_entry_lite.h +++ b/src/google/protobuf/map_entry_lite.h @@ -214,7 +214,7 @@ class MapEntryImpl : public Base { ctx->SetLastTag(tag); return ptr; } - ptr = UnknownFieldParse(tag, nullptr, ptr, ctx); + ptr = UnknownFieldParse(tag, static_cast(nullptr), ptr, ctx); } GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); } diff --git a/src/google/protobuf/map_field.cc b/src/google/protobuf/map_field.cc index a0c96df7ed..5ed84f3698 100644 --- a/src/google/protobuf/map_field.cc +++ b/src/google/protobuf/map_field.cc @@ -33,6 +33,8 @@ #include +#include + namespace google { namespace protobuf { namespace internal { diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 731f4ca167..f531a7c58e 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -34,6 +34,7 @@ // Sanjay Ghemawat, Jeff Dean, and others. #include +#include #include #include @@ -48,6 +49,7 @@ #include #include #include + #include #include @@ -60,6 +62,11 @@ std::string MessageLite::InitializationErrorString() const { return "(cannot determine missing fields for lite message)"; } +std::string MessageLite::DebugString() const { + std::uintptr_t address = reinterpret_cast(this); + return StrCat("MessageLite at 0x", strings::Hex(address)); +} + namespace { // When serializing, we first compute the byte size, then serialize the message. @@ -123,7 +130,9 @@ bool MergePartialFromImpl(StringPiece input, MessageLite* msg) { const char* ptr; internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(), aliasing, &ptr, input); - return ctx.AtLegitimateEnd(msg->_InternalParse(ptr, &ctx)); + ptr = msg->_InternalParse(ptr, &ctx); + // ctx has an explicit limit set (length of string_view). + return ptr && ctx.EndedAtLimit(); } template @@ -131,7 +140,9 @@ bool MergePartialFromImpl(io::ZeroCopyInputStream* input, MessageLite* msg) { const char* ptr; internal::ParseContext ctx(io::CodedInputStream::GetDefaultRecursionLimit(), aliasing, &ptr, input); - return ctx.AtLegitimateEnd(msg->_InternalParse(ptr, &ctx)); + ptr = msg->_InternalParse(ptr, &ctx); + // ctx has no explicit limit (hence we end on end of stream) + return ptr && ctx.EndedAtEndOfStream(); } template @@ -229,13 +240,15 @@ bool MessageLite::MergePartialFromCodedStream(io::CodedInputStream* input) { ctx.data().pool = input->GetExtensionPool(); ctx.data().factory = input->GetExtensionFactory(); ptr = _InternalParse(ptr, &ctx); - if (!ptr) return false; + if (PROTOBUF_PREDICT_FALSE(!ptr)) return false; ctx.BackUp(ptr); - if (ctx.LastTagMinus1() != 0) { - input->SetLastTag(ctx.LastTagMinus1() + 1); + if (!ctx.EndedAtEndOfStream()) { + GOOGLE_DCHECK(ctx.LastTag() != 1); // We can't end on a pushed limit. + if (ctx.IsExceedingLimit(ptr)) return false; + input->SetLastTag(ctx.LastTag()); return true; } - if (ctx.AtLimit(ptr)) input->SetConsumed(); + input->SetConsumed(); return true; } #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index ba9abf7239..8deb14343a 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -234,6 +234,20 @@ class PROTOBUF_EXPORT MessageLite { // results are undefined (probably crash). virtual void CheckTypeAndMergeFrom(const MessageLite& other) = 0; + // These methods return a human-readable summary of the message. Note that + // since the MessageLite interface does not support reflection, there is very + // little information that these methods can provide. They are shadowed by + // methods of the same name on the Message interface which provide much more + // information. The methods here are intended primarily to facilitate code + // reuse for logic that needs to interoperate with both full and lite protos. + // + // The format of the returned string is subject to change, so please do not + // assume it will remain stable over time. + std::string DebugString() const; + std::string ShortDebugString() const { + return DebugString(); + } + // Parsing --------------------------------------------------------- // Methods for parsing in protocol buffer format. Most of these are // just simple wrappers around MergeFromCodedStream(). Clear() will be diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index abf7caf864..2aec32e98e 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -157,11 +157,15 @@ std::pair EpsCopyInputStream::DoneFallback(const char* ptr, GOOGLE_DCHECK(ptr >= limit_end_); int overrun = ptr - buffer_end_; GOOGLE_DCHECK(overrun <= kSlopBytes); // Guaranteed by parse loop. - GOOGLE_DCHECK(overrun != limit_); - // We either exceeded the limit (parse error) or we need to get new data. - // Did we exceed the limit? Is so parse error. + // Did we exceeded the limit (parse error). if (PROTOBUF_PREDICT_FALSE(overrun > limit_)) return {nullptr, true}; - GOOGLE_DCHECK(overrun <= limit_); // Follows from above + GOOGLE_DCHECK(overrun != limit_); // Guaranteed by caller. + GOOGLE_DCHECK(overrun < limit_); // Follows from above + // TODO(gerbens) Instead of this dcheck we could just assign, and remove + // updating the limit_end from PopLimit, ie. + // limit_end_ = buffer_end_ + (std::min)(0, limit_); + // if (ptr < limit_end_) return {ptr, false}; + GOOGLE_DCHECK(limit_end_ == buffer_end_ + (std::min)(0, limit_)); // At this point we know the following assertion holds. GOOGLE_DCHECK(limit_ > 0); GOOGLE_DCHECK(limit_end_ == buffer_end_); // because limit_ > 0 @@ -174,6 +178,8 @@ std::pair EpsCopyInputStream::DoneFallback(const char* ptr, if (PROTOBUF_PREDICT_FALSE(overrun != 0)) return {nullptr, true}; GOOGLE_DCHECK(limit_ > 0); limit_end_ = buffer_end_; + // Distinquish ending on a pushed limit or ending on end-of-stream. + SetEndOfStream(); return {ptr, true}; } limit_ -= buffer_end_ - p; // Adjust limit_ relative to new anchor @@ -193,12 +199,19 @@ const char* EpsCopyInputStream::ReadStringFallback(const char* ptr, int size, s->clear(); // TODO(gerbens) assess security. At the moment its parity with // CodedInputStream but it allows a payload to reserve large memory. - if (size <= buffer_end_ - ptr + limit_) s->reserve(size); + if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) { + s->reserve(size); + } return AppendStringFallback(ptr, size, s); } const char* EpsCopyInputStream::AppendStringFallback(const char* ptr, int size, std::string* str) { + // TODO(gerbens) assess security. At the moment its parity with + // CodedInputStream but it allows a payload to reserve large memory. + if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) { + str->reserve(size); + } return AppendSize(ptr, size, [str](const char* p, int s) { str->append(p, s); }); } @@ -264,7 +277,6 @@ const char* EpsCopyInputStream::InitFrom(io::ZeroCopyInputStream* zcis) { } next_chunk_ = nullptr; size_ = 0; - limit_ = 0; limit_end_ = buffer_end_ = buffer_; return buffer_; } @@ -340,6 +352,13 @@ bool VerifyUTF8(StringPiece str, const char* field_name) { return true; } +const char* InlineGreedyStringParser(std::string* s, const char* ptr, + ParseContext* ctx) { + int size = ReadSize(&ptr); + if (!ptr) return nullptr; + return ctx->ReadString(ptr, size, s); +} + const char* InlineGreedyStringParserUTF8(std::string* s, const char* ptr, ParseContext* ctx, const char* field_name) { @@ -348,16 +367,6 @@ const char* InlineGreedyStringParserUTF8(std::string* s, const char* ptr, return p; } -const char* InlineGreedyStringParserUTF8Verify(std::string* s, const char* ptr, - ParseContext* ctx, - const char* field_name) { - auto p = InlineGreedyStringParser(s, ptr, ctx); -#ifndef NDEBUG - VerifyUTF8(*s, field_name); -#endif // !NDEBUG - return p; -} - template const char* VarintParser(void* object, const char* ptr, ParseContext* ctx) { @@ -531,6 +540,12 @@ const char* UnknownFieldParse(uint32 tag, std::string* unknown, const char* ptr, return FieldParser(tag, field_parser, ptr, ctx); } +const char* UnknownFieldParse(uint32 tag, + InternalMetadataWithArenaLite* metadata, + const char* ptr, ParseContext* ctx) { + return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx); +} + } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 82756e3377..1bf6100c42 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -124,17 +125,17 @@ class PROTOBUF_EXPORT EpsCopyInputStream { PROTOBUF_MUST_USE_RESULT int PushLimit(const char* ptr, int limit) { GOOGLE_DCHECK(limit >= 0); limit += ptr - buffer_end_; - if (limit < 0) limit_end_ = buffer_end_ + limit; + limit_end_ = buffer_end_ + (std::min)(0, limit); auto old_limit = limit_; limit_ = limit; return old_limit - limit; } - PROTOBUF_MUST_USE_RESULT bool PopLimit(int delta, const char* ptr) { - // Ensure not to forget to check PushLimit return value - GOOGLE_DCHECK(delta >= 0); - if (ptr == nullptr || ptr - buffer_end_ != limit_) return false; + PROTOBUF_MUST_USE_RESULT bool PopLimit(int delta) { + if (PROTOBUF_PREDICT_FALSE(!EndedAtLimit())) return false; limit_ = limit_ + delta; + // TODO(gerbens) We could remove this line and hoist the code to + // DoneFallback. Study the perf/bin-size effects. limit_end_ = buffer_end_ + (std::min)(0, limit_); return true; } @@ -175,9 +176,19 @@ class PROTOBUF_EXPORT EpsCopyInputStream { PROTOBUF_MUST_USE_RESULT const char* ReadPackedVarint(const char* ptr, Add add); - bool AtLimit(const char* ptr) const { - return (ptr - buffer_end_ == limit_) || - (next_chunk_ == nullptr && limit_ > 0 && ptr == buffer_end_); + uint32 LastTag() const { return last_tag_minus_1_ + 1; } + bool ConsumeEndGroup(uint32 start_tag) { + bool res = last_tag_minus_1_ == start_tag; + last_tag_minus_1_ = 0; + return res; + } + bool EndedAtLimit() const { return last_tag_minus_1_ == 0; } + bool EndedAtEndOfStream() const { return last_tag_minus_1_ == 1; } + void SetLastTag(uint32 tag) { last_tag_minus_1_ = tag - 1; } + void SetEndOfStream() { last_tag_minus_1_ = 1; } + bool IsExceedingLimit(const char* ptr) { + return ptr > limit_end_ && + (next_chunk_ == nullptr || ptr - buffer_end_ > limit_); } protected: @@ -233,6 +244,20 @@ class PROTOBUF_EXPORT EpsCopyInputStream { char buffer_[2 * kSlopBytes] = {}; enum { kNoAliasing = 0, kOnPatch = 1, kNoDelta = 2 }; std::uintptr_t aliasing_ = kNoAliasing; + // This variable is used to communicate how the parse ended, in order to + // completely verify the parsed data. A wire-format parse can end because of + // one of the following conditions: + // 1) A parse can end on a pushed limit. + // 2) A parse can end on End Of Stream (EOS). + // 3) A parse can end on 0 tag (only valid for toplevel message). + // 4) A parse can end on an end-group tag. + // This variable should always be set to 0, which indicates case 1. If the + // parse terminated due to EOS (case 2), it's set to 1. In case the parse + // ended due to a terminating tag (case 3 and 4) it's set to (tag - 1). + // This var doesn't really belong in EpsCopyInputStream and should be part of + // the ParseContext, but case 2 is most easily and optimally implemented in + // DoneFallback. + uint32 last_tag_minus_1_ = 0; std::pair DoneFallback(const char* ptr, int d); const char* Next(int overrun, int d); @@ -306,12 +331,6 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { bool DoneNoSlopCheck(const char** ptr) { return DoneWithCheck(ptr, -1); } int depth() const { return depth_; } - void SetLastTag(uint32 tag) { last_tag_minus_1_ = tag - 1; } - uint32 LastTagMinus1() const { return last_tag_minus_1_; } - - bool AtLegitimateEnd(const char* ptr) const { - return ptr && AtLimit(ptr) && last_tag_minus_1_ == 0; - } Data& data() { return data_; } const Data& data() const { return data_; } @@ -331,8 +350,7 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { ptr = msg->_InternalParse(ptr, this); group_depth_--; depth_++; - if (last_tag_minus_1_ != tag) return nullptr; - last_tag_minus_1_ = 0; + if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr; return ptr; } @@ -346,7 +364,6 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { // Unfortunately necessary for the fringe case of ending on 0 or end-group tag // in the last kSlopBytes of a ZeroCopyInputStream chunk. int group_depth_ = INT_MIN; - uint32 last_tag_minus_1_ = 0; Data data_; }; @@ -403,12 +420,26 @@ inline uint32 DecodeTwoBytes(uint32 value, const char** ptr) { // Used for tags, could read up to 5 bytes which must be available. // Caller must ensure its safe to call. -inline const char* ReadTag(const char* p, uint32* out) { - return VarintParse<5>(p, out); -} std::pair ReadTagFallback(const char* p, uint32 res); +inline const char* ReadTag(const char* p, uint32* out) { + uint32 res = static_cast(p[0]); + if (res < 128) { + *out = res; + return p + 1; + } + uint32 second = static_cast(p[1]); + res += (second - 1) << 7; + if (second < 128) { + *out = res; + return p + 2; + } + auto tmp = ReadTagFallback(p + 2, res); + *out = tmp.second; + return tmp.first; +} + // Will preload the next 2 bytes inline const char* ReadTag(const char* p, uint32* out, uint32* preload) { uint32 res = static_cast(p[0]); @@ -536,10 +567,11 @@ PROTOBUF_MUST_USE_RESULT const char* ParseContext::ParseMessage( int size = ReadSize(&ptr); if (!ptr) return nullptr; auto old = PushLimit(ptr, size); - if (--depth_ < 0 || old < 0) return nullptr; + if (--depth_ < 0) return nullptr; ptr = msg->_InternalParse(ptr, this); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) return nullptr; depth_++; - if (!PopLimit(old, ptr) || last_tag_minus_1_ != 0) return nullptr; + if (!PopLimit(old)) return nullptr; return ptr; } @@ -555,7 +587,7 @@ const char* EpsCopyInputStream::ReadPackedVarint(const char* ptr, Add add) { if (!ptr) return nullptr; add(varint); } - if (!PopLimit(old, ptr)) return nullptr; + if (!PopLimit(old)) return nullptr; return ptr; } @@ -564,19 +596,22 @@ PROTOBUF_EXPORT bool VerifyUTF8(StringPiece s, const char* field_name); // All the string parsers with or without UTF checking and for all CTypes. -inline PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParser( - std::string* s, const char* ptr, ParseContext* ctx) { - int size = ReadSize(&ptr); - if (!ptr) return nullptr; - return ctx->ReadString(ptr, size, s); -} +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParser( + std::string* s, const char* ptr, ParseContext* ctx); PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParserUTF8(std::string* s, const char* ptr, ParseContext* ctx, const char* field_name); -PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* -InlineGreedyStringParserUTF8Verify(std::string* s, const char* ptr, - ParseContext* ctx, const char* field_name); +// Inline because we don't want to pay the price of field_name in opt mode. +inline PROTOBUF_MUST_USE_RESULT const char* InlineGreedyStringParserUTF8Verify( + std::string* s, const char* ptr, ParseContext* ctx, + const char* field_name) { + auto p = InlineGreedyStringParser(s, ptr, ctx); +#ifndef NDEBUG + VerifyUTF8(*s, field_name); +#endif // !NDEBUG + return p; +} // Add any of the following lines to debug which parse function is failing. @@ -705,6 +740,9 @@ PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownGroupLiteParse( // UnknownFieldSet* to make the generated code isomorphic between full and lite. PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownFieldParse( uint32 tag, std::string* unknown, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT PROTOBUF_MUST_USE_RESULT const char* UnknownFieldParse( + uint32 tag, InternalMetadataWithArenaLite* metadata, const char* ptr, + ParseContext* ctx); } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/reflection_ops.cc b/src/google/protobuf/reflection_ops.cc index f1eea1f240..f8f9dc3f24 100644 --- a/src/google/protobuf/reflection_ops.cc +++ b/src/google/protobuf/reflection_ops.cc @@ -46,6 +46,8 @@ #include +#include + namespace google { namespace protobuf { namespace internal { diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 0515e01441..2552e232c2 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -2282,14 +2282,23 @@ namespace internal { // This code based on net/proto/proto-array-internal.h by Jeffrey Yasskin // (jyasskin@google.com). template -class RepeatedPtrIterator { +class RepeatedPtrIterator + : public std::iterator< + std::random_access_iterator_tag, Element> { public: typedef RepeatedPtrIterator iterator; - typedef std::random_access_iterator_tag iterator_category; + typedef std::iterator< + std::random_access_iterator_tag, Element> superclass; + + // Shadow the value_type in std::iterator<> because const_iterator::value_type + // needs to be T, not const T. typedef typename std::remove_const::type value_type; - typedef std::ptrdiff_t difference_type; - typedef Element* pointer; - typedef Element& reference; + + // Let the compiler know that these are type names, so we don't have to + // write "typename" in front of them everywhere. + typedef typename superclass::reference reference; + typedef typename superclass::pointer pointer; + typedef typename superclass::difference_type difference_type; RepeatedPtrIterator() : it_(NULL) {} explicit RepeatedPtrIterator(void* const* it) : it_(it) {} @@ -2369,14 +2378,21 @@ class RepeatedPtrIterator { // referenced by the iterator. It should either be "void *" for a mutable // iterator, or "const void* const" for a constant iterator. template -class RepeatedPtrOverPtrsIterator { +class RepeatedPtrOverPtrsIterator + : public std::iterator { public: typedef RepeatedPtrOverPtrsIterator iterator; - typedef std::random_access_iterator_tag iterator_category; + typedef std::iterator superclass; + + // Shadow the value_type in std::iterator<> because const_iterator::value_type + // needs to be T, not const T. typedef typename std::remove_const::type value_type; - typedef std::ptrdiff_t difference_type; - typedef Element* pointer; - typedef Element& reference; + + // Let the compiler know that these are type names, so we don't have to + // write "typename" in front of them everywhere. + typedef typename superclass::reference reference; + typedef typename superclass::pointer pointer; + typedef typename superclass::difference_type difference_type; RepeatedPtrOverPtrsIterator() : it_(NULL) {} explicit RepeatedPtrOverPtrsIterator(VoidPtr* it) : it_(it) {} diff --git a/src/google/protobuf/test_messages_proto2.proto b/src/google/protobuf/test_messages_proto2.proto index 60dbfc753a..dc6aaa378a 100644 --- a/src/google/protobuf/test_messages_proto2.proto +++ b/src/google/protobuf/test_messages_proto2.proto @@ -33,6 +33,8 @@ // - conformance tests // +// LINT: ALLOW_GROUPS + syntax = "proto2"; package protobuf_test_messages.proto2; @@ -180,6 +182,9 @@ message TestAllTypesProto2 { optional int32 field_name17__ = 417; optional int32 Field_name18__ = 418; + // Reserved for unknown fields test. + reserved 1000 to 9999; + // message_set test case. message MessageSetCorrect { option message_set_wire_format = true; @@ -214,3 +219,15 @@ enum ForeignEnumProto2 { extend TestAllTypesProto2 { optional int32 extension_int32 = 120; } + +message UnknownToTestAllTypes { + optional int32 optional_int32 = 1001; + optional string optional_string = 1002; + optional ForeignMessageProto2 nested_message = 1003; + optional group OptionalGroup = 1004 { + optional int32 a = 1; + } + optional bool optional_bool = 1006; + repeated int32 repeated_int32 = 1011; +} + diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 34e442cd6b..ba0c3028ee 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -64,6 +64,8 @@ #include #include +#include + namespace google { namespace protobuf { diff --git a/src/google/protobuf/unknown_field_set.cc b/src/google/protobuf/unknown_field_set.cc index 190755357f..2f8b763681 100644 --- a/src/google/protobuf/unknown_field_set.cc +++ b/src/google/protobuf/unknown_field_set.cc @@ -357,6 +357,11 @@ const char* UnknownFieldParse(uint64 tag, UnknownFieldSet* unknown, return FieldParser(tag, field_parser, ptr, ctx); } +const char* UnknownFieldParse(uint32 tag, InternalMetadataWithArena* metadata, + const char* ptr, ParseContext* ctx) { + return UnknownFieldParse(tag, metadata->mutable_unknown_fields(), ptr, ctx); +} + } // namespace internal #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER diff --git a/src/google/protobuf/unknown_field_set.h b/src/google/protobuf/unknown_field_set.h index 17b8c077d4..0d61f89358 100644 --- a/src/google/protobuf/unknown_field_set.h +++ b/src/google/protobuf/unknown_field_set.h @@ -212,6 +212,9 @@ const char* UnknownGroupParse(UnknownFieldSet* unknown, const char* ptr, PROTOBUF_EXPORT const char* UnknownFieldParse(uint64 tag, UnknownFieldSet* unknown, const char* ptr, ParseContext* ctx); +PROTOBUF_EXPORT +const char* UnknownFieldParse(uint32 tag, InternalMetadataWithArena* metadata, + const char* ptr, ParseContext* ctx); } // namespace internal #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER diff --git a/src/google/protobuf/util/json_format.proto b/src/google/protobuf/util/json_format.proto index 44f18f3ade..d773267216 100644 --- a/src/google/protobuf/util/json_format.proto +++ b/src/google/protobuf/util/json_format.proto @@ -35,8 +35,8 @@ // A proto file we will use for unit testing. syntax = "proto2"; -package protobuf_unittest; +package protobuf_unittest; message TestFlagsAndStrings { required int32 A = 1; diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index 28e593df87..1e72794d45 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -32,17 +32,17 @@ syntax = "proto3"; package proto3; -option java_package = "com.google.protobuf.util"; -option java_outer_classname = "JsonFormatProto3"; - +import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/any.proto"; -import "google/protobuf/field_mask.proto"; import "google/protobuf/unittest.proto"; +option java_package = "com.google.protobuf.util"; +option java_outer_classname = "JsonFormatProto3"; + enum EnumType { FOO = 0; BAR = 1; diff --git a/src/google/protobuf/util/message_differencer.h b/src/google/protobuf/util/message_differencer.h index 1c7451e02c..7504227622 100644 --- a/src/google/protobuf/util/message_differencer.h +++ b/src/google/protobuf/util/message_differencer.h @@ -127,7 +127,7 @@ class PROTOBUF_EXPORT MessageDifferencer { // defined as all fields within the two messages having the same value. This // differs from the Equals method above in that fields with default values // are considered set to said value automatically. For details on how default - // values are defined for each field type, see + // values are defined for each field type, see: // https://developers.google.com/protocol-buffers/docs/proto?csw=1#optional. // Also, Equivalent() ignores unknown fields. Use IgnoreField() and Compare() // if some fields should be ignored in the comparison. diff --git a/src/google/protobuf/util/message_differencer_unittest.proto b/src/google/protobuf/util/message_differencer_unittest.proto index 698775f14c..fafd19cd75 100644 --- a/src/google/protobuf/util/message_differencer_unittest.proto +++ b/src/google/protobuf/util/message_differencer_unittest.proto @@ -35,6 +35,7 @@ // This file contains messages for testing repeated field comparison syntax = "proto2"; + package protobuf_unittest; option optimize_for = SPEED; @@ -53,22 +54,21 @@ message TestField { message TestDiffMessage { repeated group Item = 1 { - optional int32 a = 2; // Test basic repeated field comparison. - optional string b = 4; // Test basic repeated field comparison. - repeated int32 ra = 3; // Test SetOfSet Comparison. - repeated string rb = 5; // Test TreatAsMap when key is repeated - optional TestField m = 6; // Test TreatAsMap when key is a message - repeated TestField rm = 7; // Test TreatAsMap when key is a repeated - // message + optional int32 a = 2; // Test basic repeated field comparison. + optional string b = 4; // Test basic repeated field comparison. + repeated int32 ra = 3; // Test SetOfSet Comparison. + repeated string rb = 5; // Test TreatAsMap when key is repeated + optional TestField m = 6; // Test TreatAsMap when key is a message + repeated TestField rm = 7; // Test TreatAsMap when key is a repeated + // message } - optional int32 v = 13 [deprecated = true]; - optional string w = 14; - optional TestField m = 15; - repeated int32 rv = 11; // Test for combinations - repeated string rw = 10; // Test for combinations - repeated TestField rm = 12 [deprecated = true]; // Test for combinations + optional int32 v = 13 [deprecated = true]; + optional string w = 14; + optional TestField m = 15; + repeated int32 rv = 11; // Test for combinations + repeated string rw = 10; // Test for combinations + repeated TestField rm = 12 [deprecated = true]; // Test for combinations extensions 100 to 199; } - diff --git a/src/google/protobuf/wire_format.cc b/src/google/protobuf/wire_format.cc index 6c8c6b764d..7a25207fdc 100644 --- a/src/google/protobuf/wire_format.cc +++ b/src/google/protobuf/wire_format.cc @@ -52,6 +52,8 @@ #include +#include + const size_t kMapEntryTagByteSize = 2; namespace google { diff --git a/src/google/protobuf/wire_format_lite.h b/src/google/protobuf/wire_format_lite.h index b8ae08e745..2468163158 100644 --- a/src/google/protobuf/wire_format_lite.h +++ b/src/google/protobuf/wire_format_lite.h @@ -248,16 +248,6 @@ class PROTOBUF_EXPORT WireFormatLite { // of these methods are defined in wire_format_lite_inl.h; you must #include // that file to use these. -#ifdef NDEBUG -#define INL PROTOBUF_ALWAYS_INLINE -#else -// Avoid excessive inlining in non-optimized builds. Without other optimizations -// the inlining is not going to provide benefits anyway and the huge resulting -// functions, especially in the proto-generated serialization functions, produce -// stack frames so large that many tests run into stack overflows (b/32192897). -#define INL -#endif - // Read fields, not including tags. The assumption is that you already // read the tag to determine what field to read. @@ -265,15 +255,16 @@ class PROTOBUF_EXPORT WireFormatLite { // the represented type and the FieldType. These are specialized with the // appropriate definition for each declared type. template - INL static bool ReadPrimitive(io::CodedInputStream* input, CType* value); + PROTOBUF_ALWAYS_INLINE static bool ReadPrimitive(io::CodedInputStream* input, + CType* value); // Reads repeated primitive values, with optimizations for repeats. // tag_size and tag should both be compile-time constants provided by the // protocol compiler. template - INL static bool ReadRepeatedPrimitive(int tag_size, uint32 tag, - io::CodedInputStream* input, - RepeatedField* value); + PROTOBUF_ALWAYS_INLINE static bool ReadRepeatedPrimitive( + int tag_size, uint32 tag, io::CodedInputStream* input, + RepeatedField* value); // Identical to ReadRepeatedPrimitive, except will not inline the // implementation. @@ -288,15 +279,15 @@ class PROTOBUF_EXPORT WireFormatLite { // This is only implemented for the types with fixed wire size, e.g. // float, double, and the (s)fixed* types. template - INL static const uint8* ReadPrimitiveFromArray(const uint8* buffer, - CType* value); + PROTOBUF_ALWAYS_INLINE static const uint8* ReadPrimitiveFromArray( + const uint8* buffer, CType* value); // Reads a primitive packed field. // // This is only implemented for packable types. template - INL static bool ReadPackedPrimitive(io::CodedInputStream* input, - RepeatedField* value); + PROTOBUF_ALWAYS_INLINE static bool ReadPackedPrimitive( + io::CodedInputStream* input, RepeatedField* value); // Identical to ReadPackedPrimitive, except will not inline the // implementation. @@ -364,28 +355,38 @@ class PROTOBUF_EXPORT WireFormatLite { // Write a tag. The Write*() functions typically include the tag, so // normally there's no need to call this unless using the Write*NoTag() // variants. - INL static void WriteTag(int field_number, WireType type, - io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteTag(int field_number, WireType type, + io::CodedOutputStream* output); // Write fields, without tags. - INL static void WriteInt32NoTag(int32 value, io::CodedOutputStream* output); - INL static void WriteInt64NoTag(int64 value, io::CodedOutputStream* output); - INL static void WriteUInt32NoTag(uint32 value, io::CodedOutputStream* output); - INL static void WriteUInt64NoTag(uint64 value, io::CodedOutputStream* output); - INL static void WriteSInt32NoTag(int32 value, io::CodedOutputStream* output); - INL static void WriteSInt64NoTag(int64 value, io::CodedOutputStream* output); - INL static void WriteFixed32NoTag(uint32 value, - io::CodedOutputStream* output); - INL static void WriteFixed64NoTag(uint64 value, - io::CodedOutputStream* output); - INL static void WriteSFixed32NoTag(int32 value, - io::CodedOutputStream* output); - INL static void WriteSFixed64NoTag(int64 value, - io::CodedOutputStream* output); - INL static void WriteFloatNoTag(float value, io::CodedOutputStream* output); - INL static void WriteDoubleNoTag(double value, io::CodedOutputStream* output); - INL static void WriteBoolNoTag(bool value, io::CodedOutputStream* output); - INL static void WriteEnumNoTag(int value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteInt32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteInt64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteUInt32NoTag( + uint32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteUInt64NoTag( + uint64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSInt32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSInt64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFixed32NoTag( + uint32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFixed64NoTag( + uint64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSFixed32NoTag( + int32 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteSFixed64NoTag( + int64 value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteFloatNoTag( + float value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteDoubleNoTag( + double value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteBoolNoTag( + bool value, io::CodedOutputStream* output); + PROTOBUF_ALWAYS_INLINE static void WriteEnumNoTag( + int value, io::CodedOutputStream* output); // Write array of primitive fields, without tags static void WriteFloatArray(const float* a, int n, @@ -468,147 +469,161 @@ class PROTOBUF_EXPORT WireFormatLite { io::CodedOutputStream* output); // Like above, but use only *ToArray methods of CodedOutputStream. - INL static uint8* WriteTagToArray(int field_number, WireType type, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteTagToArray(int field_number, + WireType type, + uint8* target); // Write fields, without tags. - INL static uint8* WriteInt32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteInt64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteUInt32NoTagToArray(uint32 value, uint8* target); - INL static uint8* WriteUInt64NoTagToArray(uint64 value, uint8* target); - INL static uint8* WriteSInt32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteSInt64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteFixed32NoTagToArray(uint32 value, uint8* target); - INL static uint8* WriteFixed64NoTagToArray(uint64 value, uint8* target); - INL static uint8* WriteSFixed32NoTagToArray(int32 value, uint8* target); - INL static uint8* WriteSFixed64NoTagToArray(int64 value, uint8* target); - INL static uint8* WriteFloatNoTagToArray(float value, uint8* target); - INL static uint8* WriteDoubleNoTagToArray(double value, uint8* target); - INL static uint8* WriteBoolNoTagToArray(bool value, uint8* target); - INL static uint8* WriteEnumNoTagToArray(int value, uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray(uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray(uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray(uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray(uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray(int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray(int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray(float value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray(double value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray(bool value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray(int value, + uint8* target); // Write fields, without tags. These require that value.size() > 0. template - INL static uint8* WritePrimitiveNoTagToArray(const RepeatedField& value, - uint8* (*Writer)(T, uint8*), - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), + uint8* target); template - INL static uint8* WriteFixedNoTagToArray(const RepeatedField& value, - uint8* (*Writer)(T, uint8*), - uint8* target); - - INL static uint8* WriteInt32NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteInt64NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteUInt32NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteUInt64NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteSInt32NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteSInt64NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteFixed32NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteFixed64NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteSFixed32NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteSFixed64NoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteFloatNoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteDoubleNoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteBoolNoTagToArray(const RepeatedField& value, - uint8* output); - INL static uint8* WriteEnumNoTagToArray(const RepeatedField& value, - uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixedNoTagToArray( + const RepeatedField& value, uint8* (*Writer)(T, uint8*), + uint8* target); + + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64NoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatNoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleNoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolNoTagToArray( + const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumNoTagToArray( + const RepeatedField& value, uint8* output); // Write fields, including tags. - INL static uint8* WriteInt32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteInt64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteUInt32ToArray(int field_number, uint32 value, - uint8* target); - INL static uint8* WriteUInt64ToArray(int field_number, uint64 value, - uint8* target); - INL static uint8* WriteSInt32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteSInt64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteFixed32ToArray(int field_number, uint32 value, - uint8* target); - INL static uint8* WriteFixed64ToArray(int field_number, uint64 value, - uint8* target); - INL static uint8* WriteSFixed32ToArray(int field_number, int32 value, - uint8* target); - INL static uint8* WriteSFixed64ToArray(int field_number, int64 value, - uint8* target); - INL static uint8* WriteFloatToArray(int field_number, float value, - uint8* target); - INL static uint8* WriteDoubleToArray(int field_number, double value, - uint8* target); - INL static uint8* WriteBoolToArray(int field_number, bool value, - uint8* target); - INL static uint8* WriteEnumToArray(int field_number, int value, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray(int field_number, + uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray(int field_number, + uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray(int field_number, + uint32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray(int field_number, + uint64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray(int field_number, + int32 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray(int field_number, + int64 value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray(int field_number, + float value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray(int field_number, + double value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray(int field_number, + bool value, + uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray(int field_number, + int value, + uint8* target); template - INL static uint8* WritePrimitiveToArray(int field_number, - const RepeatedField& value, - uint8* (*Writer)(int, T, uint8*), - uint8* target); - - INL static uint8* WriteInt32ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteInt64ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteUInt32ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteUInt64ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteSInt32ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteSInt64ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteFixed32ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteFixed64ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteSFixed32ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteSFixed64ToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteFloatToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteDoubleToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteBoolToArray(int field_number, - const RepeatedField& value, - uint8* output); - INL static uint8* WriteEnumToArray(int field_number, - const RepeatedField& value, - uint8* output); - - INL static uint8* WriteStringToArray(int field_number, - const std::string& value, uint8* target); - INL static uint8* WriteBytesToArray(int field_number, - const std::string& value, uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WritePrimitiveToArray( + int field_number, const RepeatedField& value, + uint8* (*Writer)(int, T, uint8*), uint8* target); + + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteInt64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteUInt64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSInt64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFixed64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed32ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteSFixed64ToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteFloatToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteDoubleToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBoolToArray( + int field_number, const RepeatedField& value, uint8* output); + PROTOBUF_ALWAYS_INLINE static uint8* WriteEnumToArray( + int field_number, const RepeatedField& value, uint8* output); + + PROTOBUF_ALWAYS_INLINE static uint8* WriteStringToArray( + int field_number, const std::string& value, uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* WriteBytesToArray( + int field_number, const std::string& value, uint8* target); // Whether to serialize deterministically (e.g., map keys are // sorted) is a property of a CodedOutputStream, and in the process @@ -616,39 +631,33 @@ class PROTOBUF_EXPORT WireFormatLite { // have a CodedOutputStream available, so they get an additional parameter // telling them whether to serialize deterministically. template - INL static uint8* InternalWriteGroupToArray(int field_number, - const MessageType& value, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroupToArray( + int field_number, const MessageType& value, uint8* target); template - INL static uint8* InternalWriteMessageToArray(int field_number, - const MessageType& value, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessageToArray( + int field_number, const MessageType& value, uint8* target); // Like above, but de-virtualize the call to SerializeWithCachedSizes(). The // pointer must point at an instance of MessageType, *not* a subclass (or // the subclass must not override SerializeWithCachedSizes()). template - INL static uint8* InternalWriteGroupNoVirtualToArray(int field_number, - const MessageType& value, - uint8* target); + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteGroupNoVirtualToArray( + int field_number, const MessageType& value, uint8* target); template - INL static uint8* InternalWriteMessageNoVirtualToArray( + PROTOBUF_ALWAYS_INLINE static uint8* InternalWriteMessageNoVirtualToArray( int field_number, const MessageType& value, uint8* target); // For backward-compatibility, the last four methods also have versions // that are non-deterministic always. - INL static uint8* WriteGroupToArray(int field_number, - const MessageLite& value, uint8* target) { + PROTOBUF_ALWAYS_INLINE static uint8* WriteGroupToArray( + int field_number, const MessageLite& value, uint8* target) { return InternalWriteGroupToArray(field_number, value, target); } - INL static uint8* WriteMessageToArray(int field_number, - const MessageLite& value, - uint8* target) { + PROTOBUF_ALWAYS_INLINE static uint8* WriteMessageToArray( + int field_number, const MessageLite& value, uint8* target) { return InternalWriteMessageToArray(field_number, value, target); } -#undef INL - // Compute the byte size of a field. The XxSize() functions do NOT include // the tag, so you must also call TagSize(). (This is because, for repeated // fields, you should only call TagSize() once and multiply it by the element diff --git a/src/google/protobuf/wire_format_lite_inl.h b/src/google/protobuf/wire_format_lite_inl.h deleted file mode 100644 index 8b4522cd42..0000000000 --- a/src/google/protobuf/wire_format_lite_inl.h +++ /dev/null @@ -1,38 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__ -#define GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__ - -// Please include wire_format_lite.h instead. - -#include - -#endif // GOOGLE_PROTOBUF_WIRE_FORMAT_LITE_INL_H__