From a3e1de01130aa665be3d559f69bbed8a487f2f9d Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Fri, 17 Jun 2022 04:13:20 -0700 Subject: [PATCH 01/11] Sync from Piper @455586341 PROTOBUF_SYNC_PIPER --- BUILD.bazel | 2 +- CHANGES.txt | 6 + conformance/conformance_cpp.cc | 271 +++--- csharp/.gitignore | 1 + .../Google.Protobuf.Test/ByteStringTest.cs | 4 - .../Collections/MapFieldTest.cs | 2 - .../Compatibility/StreamExtensionsTest.cs | 67 -- .../Compatibility/TypeExtensionsTest.cs | 2 - .../Google.Protobuf.Test/FieldCodecTest.cs | 2 - csharp/src/Google.Protobuf/ByteString.cs | 7 - csharp/src/Google.Protobuf/ByteStringAsync.cs | 2 - .../Google.Protobuf/Collections/MapField.cs | 8 +- .../Collections/ReadOnlyDictionary.cs | 147 ---- .../Collections/RepeatedField.cs | 5 +- .../Compatibility/PropertyInfoExtensions.cs | 8 - .../Compatibility/StreamExtensions.cs | 66 -- .../Compatibility/TypeExtensions.cs | 2 - .../Reflection/MessageDescriptor.cs | 4 - .../WellKnownTypes/FieldMaskPartial.cs | 5 - docs/options.md | 4 + .../google/protobuf/GeneratedMessageV3.java | 802 +++++++++--------- .../com/google/protobuf/DescriptorsTest.java | 19 +- .../google/protobuf/GeneratedMessageTest.java | 2 +- objectivec/README.md | 25 +- php/src/Google/Protobuf/FieldDescriptor.php | 35 +- .../Protobuf/Internal/FieldDescriptor.php | 38 +- .../Protobuf/Internal/OneofDescriptor.php | 9 + php/src/Google/Protobuf/OneofDescriptor.php | 9 +- src/Makefile.am | 2 + src/file_lists.cmake | 2 + src/google/protobuf/arena_impl.h | 21 +- src/google/protobuf/arenastring.cc | 6 +- src/google/protobuf/arenastring.h | 27 +- src/google/protobuf/arenaz_sampler.cc | 28 +- src/google/protobuf/arenaz_sampler.h | 31 +- src/google/protobuf/arenaz_sampler_test.cc | 61 +- .../protobuf/compiler/cpp/enum_field.cc | 9 +- src/google/protobuf/compiler/cpp/message.cc | 5 + .../compiler/cpp/parse_function_generator.cc | 18 +- .../protobuf/compiler/cpp/primitive_field.cc | 9 +- src/google/protobuf/compiler/java/message.cc | 38 +- src/google/protobuf/compiler/java/message.h | 4 - .../compiler/java/message_serialization.cc | 35 +- .../compiler/java/message_serialization.h | 91 ++ .../compiler/objectivec/objectivec_file.h | 4 +- .../objectivec/objectivec_generator.cc | 4 + .../compiler/objectivec/objectivec_helpers.cc | 11 +- .../compiler/objectivec/objectivec_helpers.h | 4 + src/google/protobuf/descriptor.pb.cc | 15 +- src/google/protobuf/descriptor.pb.h | 6 +- .../generated_message_tctable_lite.cc | 55 +- src/google/protobuf/generated_message_util.h | 51 +- src/google/protobuf/port_def.inc | 5 + src/google/protobuf/port_undef.inc | 1 + src/google/protobuf/struct.pb.cc | 2 + src/google/protobuf/stubs/status.cc | 9 +- src/google/protobuf/stubs/status.h | 10 +- src/google/protobuf/stubs/strutil.cc | 65 +- src/google/protobuf/stubs/strutil.h | 65 +- .../protobuf/util/internal/proto_writer.cc | 2 - src/google/protobuf/util/json_util_test.cc | 112 ++- 61 files changed, 1187 insertions(+), 1175 deletions(-) delete mode 100644 csharp/src/Google.Protobuf.Test/Compatibility/StreamExtensionsTest.cs delete mode 100644 csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs delete mode 100644 csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs rename csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs => src/google/protobuf/compiler/java/message_serialization.cc (72%) create mode 100644 src/google/protobuf/compiler/java/message_serialization.h diff --git a/BUILD.bazel b/BUILD.bazel index 96d973c0df..d8197aed19 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -22,7 +22,7 @@ exports_files(["LICENSE"]) # # java_proto_library( # name = "any_java_proto", -# deps = ["@com_google_protobuf//:any_proto], +# deps = ["@com_google_protobuf//:any_proto"], # ) ################################################################################ diff --git a/CHANGES.txt b/CHANGES.txt index 0ebbe6158d..12360dadc6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,9 @@ + +Unreleased version + * Handle reflection for message splitting. + * make metadata fields lazy. + * Extend visibility of plugin library to upb + 2022-05-27 version 21.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby) C++ diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 5782789dfd..1185880715 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -32,165 +32,176 @@ #include #include +#include +#include +#include + +#include +#include #include #include #include #include #include +#include +#include "conformance.pb.h" #include "conformance.pb.h" #include #include +#include +#include #include +#include -using conformance::ConformanceRequest; -using conformance::ConformanceResponse; -using google::protobuf::Descriptor; -using google::protobuf::DescriptorPool; -using google::protobuf::Message; -using google::protobuf::MessageFactory; -using google::protobuf::TextFormat; -using google::protobuf::util::BinaryToJsonString; -using google::protobuf::util::JsonParseOptions; -using google::protobuf::util::JsonToBinaryString; -using google::protobuf::util::NewTypeResolverForDescriptorPool; -using google::protobuf::util::TypeResolver; -using protobuf_test_messages::proto3::TestAllTypesProto3; -using protobuf_test_messages::proto2::TestAllTypesProto2; -using std::string; - -static const char kTypeUrlPrefix[] = "type.googleapis.com"; - -const char* kFailures[] = { -}; - -static string GetTypeUrl(const Descriptor* message) { - return string(kTypeUrlPrefix) + "/" + message->full_name(); -} - -int test_count = 0; -bool verbose = false; -TypeResolver* type_resolver; -string* type_url; +// Must be included last. +#include namespace google { namespace protobuf { - -using util::Status; - -bool CheckedRead(int fd, void *buf, size_t len) { - size_t ofs = 0; +namespace { +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::google::protobuf::util::BinaryToJsonString; +using ::google::protobuf::util::JsonParseOptions; +using ::google::protobuf::util::JsonToBinaryString; +using ::google::protobuf::util::NewTypeResolverForDescriptorPool; +using ::google::protobuf::util::TypeResolver; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; + +util::Status ReadFd(int fd, char* buf, size_t len) { while (len > 0) { - ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + ssize_t bytes_read = read(fd, buf, len); - if (bytes_read == 0) return false; + if (bytes_read == 0) { + return util::DataLossError("unexpected EOF"); + } if (bytes_read < 0) { - GOOGLE_LOG(FATAL) << "Error reading from test runner: " << strerror(errno); + return util::ErrnoToStatus(errno, "error reading from test runner"); } len -= bytes_read; - ofs += bytes_read; + buf += bytes_read; } - - return true; + return util::OkStatus(); } -void CheckedWrite(int fd, const void *buf, size_t len) { +util::Status WriteFd(int fd, const void* buf, size_t len) { if (write(fd, buf, len) != len) { - GOOGLE_LOG(FATAL) << "Error writing to test runner: " << strerror(errno); + return util::ErrnoToStatus(errno, "error reading to test runner"); } + return util::OkStatus(); } -void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { - Message *test_message; - google::protobuf::LinkMessageReflection(); - google::protobuf::LinkMessageReflection(); - const Descriptor *descriptor = DescriptorPool::generated_pool()->FindMessageTypeByName( - request.message_type()); - if (!descriptor) { - GOOGLE_LOG(FATAL) << "No such message type: " << request.message_type(); +class Harness { + public: + Harness() { + google::protobuf::LinkMessageReflection(); + google::protobuf::LinkMessageReflection(); + + resolver_.reset(NewTypeResolverForDescriptorPool( + "type.googleapis.com", DescriptorPool::generated_pool())); + type_url_ = StrCat("type.googleapis.com/", + TestAllTypesProto3::GetDescriptor()->full_name()); + } + + util::StatusOr RunTest( + const ConformanceRequest& request); + + // Returns Ok(true) if we're done processing requests. + util::StatusOr ServeConformanceRequest(); + + private: + bool verbose_ = false; + std::unique_ptr resolver_; + std::string type_url_; +}; + +util::StatusOr Harness::RunTest( + const ConformanceRequest& request) { + const Descriptor* descriptor = + DescriptorPool::generated_pool()->FindMessageTypeByName( + request.message_type()); + if (descriptor == nullptr) { + return util::NotFoundError( + StrCat("No such message type: ", request.message_type())); } - test_message = MessageFactory::generated_factory()->GetPrototype(descriptor)->New(); + + std::unique_ptr test_message( + MessageFactory::generated_factory()->GetPrototype(descriptor)->New()); + ConformanceResponse response; switch (request.payload_case()) { case ConformanceRequest::kProtobufPayload: { if (!test_message->ParseFromString(request.protobuf_payload())) { - // Getting parse details would involve something like: - // http://stackoverflow.com/questions/22121922/how-can-i-get-more-details-about-errors-generated-during-protobuf-parsing-c - response->set_parse_error("Parse error (no more details available)."); - return; + response.set_parse_error("parse error (no more details available)"); + return response; } break; } case ConformanceRequest::kJsonPayload: { - string proto_binary; JsonParseOptions options; options.ignore_unknown_fields = (request.test_category() == - conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + + std::string proto_binary; util::Status status = - JsonToBinaryString(type_resolver, *type_url, request.json_payload(), + JsonToBinaryString(resolver_.get(), type_url_, request.json_payload(), &proto_binary, options); if (!status.ok()) { - response->set_parse_error(string("Parse error: ") + - std::string(status.message())); - return; + response.set_parse_error( + StrCat("parse error: ", status.message())); + return response; } if (!test_message->ParseFromString(proto_binary)) { - response->set_runtime_error( - "Parsing JSON generates invalid proto output."); - return; + response.set_runtime_error( + "parsing JSON generated invalid proto output"); + return response; } + break; } case ConformanceRequest::kTextPayload: { - if (!TextFormat::ParseFromString(request.text_payload(), test_message)) { - response->set_parse_error("Parse error"); - return; + if (!TextFormat::ParseFromString(request.text_payload(), + test_message.get())) { + response.set_parse_error("parse error (no more details available)"); + return response; } break; } case ConformanceRequest::PAYLOAD_NOT_SET: - GOOGLE_LOG(FATAL) << "Request didn't have payload."; - break; + return util::InvalidArgumentError("request didn't have payload"); default: - GOOGLE_LOG(FATAL) << "unknown payload type: " << request.payload_case(); - break; - } - - conformance::FailureSet failures; - if (descriptor == failures.GetDescriptor()) { - for (const char* s : kFailures) failures.add_failure(s); - test_message = &failures; + return util::InvalidArgumentError( + StrCat("unknown payload type", request.payload_case())); } switch (request.requested_output_format()) { case conformance::UNSPECIFIED: - GOOGLE_LOG(FATAL) << "Unspecified output format"; - break; + return util::InvalidArgumentError("unspecified output format"); case conformance::PROTOBUF: { - GOOGLE_CHECK(test_message->SerializeToString( - response->mutable_protobuf_payload())); + GOOGLE_CHECK( + test_message->SerializeToString(response.mutable_protobuf_payload())); break; } case conformance::JSON: { - string proto_binary; + std::string proto_binary; GOOGLE_CHECK(test_message->SerializeToString(&proto_binary)); util::Status status = - BinaryToJsonString(type_resolver, *type_url, proto_binary, - response->mutable_json_payload()); + BinaryToJsonString(resolver_.get(), type_url_, proto_binary, + response.mutable_json_payload()); if (!status.ok()) { - response->set_serialize_error( - string("Failed to serialize JSON output: ") + - std::string(status.message())); - return; + response.set_serialize_error(StrCat( + "failed to serialize JSON output: ", status.message())); } break; } @@ -199,70 +210,66 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { TextFormat::Printer printer; printer.SetHideUnknownFields(!request.print_unknown_fields()); GOOGLE_CHECK(printer.PrintToString(*test_message, - response->mutable_text_payload())); + response.mutable_text_payload())); break; } default: - GOOGLE_LOG(FATAL) << "Unknown output format: " - << request.requested_output_format(); + return util::InvalidArgumentError(StrCat( + "unknown output format", request.requested_output_format())); } -} -bool DoTestIo() { - string serialized_input; - string serialized_output; - ConformanceRequest request; - ConformanceResponse response; - uint32_t bytes; + return response; +} - if (!CheckedRead(STDIN_FILENO, &bytes, sizeof(uint32_t))) { - // EOF. - return false; +util::StatusOr Harness::ServeConformanceRequest() { + uint32_t in_len; + if (!ReadFd(STDIN_FILENO, reinterpret_cast(&in_len), sizeof(in_len)) + .ok()) { + // EOF means we're done. + return true; } - serialized_input.resize(bytes); - - if (!CheckedRead(STDIN_FILENO, (char*)serialized_input.c_str(), bytes)) { - GOOGLE_LOG(ERROR) << "Unexpected EOF on stdin. " << strerror(errno); - } + std::string serialized_input; + serialized_input.resize(in_len); + RETURN_IF_ERROR(ReadFd(STDIN_FILENO, serialized_input.data(), in_len)); - if (!request.ParseFromString(serialized_input)) { - GOOGLE_LOG(FATAL) << "Parse of ConformanceRequest proto failed."; - return false; - } + ConformanceRequest request; + GOOGLE_CHECK(request.ParseFromString(serialized_input)); - DoTest(request, &response); + util::StatusOr response = RunTest(request); + RETURN_IF_ERROR(response.status()); - response.SerializeToString(&serialized_output); + std::string serialized_output; + response->SerializeToString(&serialized_output); - bytes = serialized_output.size(); - CheckedWrite(STDOUT_FILENO, &bytes, sizeof(uint32_t)); - CheckedWrite(STDOUT_FILENO, serialized_output.c_str(), bytes); + uint32_t out_len = static_cast(serialized_output.size()); + RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, &out_len, sizeof(out_len))); + RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, serialized_output.data(), out_len)); - if (verbose) { - fprintf(stderr, "conformance-cpp: request=%s, response=%s\n", - request.ShortDebugString().c_str(), - response.ShortDebugString().c_str()); + if (verbose_) { + GOOGLE_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString() + << ", response=" << response->ShortDebugString(); } - - test_count++; - - return true; + return false; } - +} // namespace } // namespace protobuf } // namespace google int main() { - type_resolver = NewTypeResolverForDescriptorPool( - kTypeUrlPrefix, DescriptorPool::generated_pool()); - type_url = new string(GetTypeUrl(TestAllTypesProto3::descriptor())); - while (1) { - if (!google::protobuf::DoTestIo()) { - fprintf(stderr, "conformance-cpp: received EOF from test runner " - "after %d tests, exiting\n", test_count); - return 0; + google::protobuf::Harness harness; + int total_runs = 0; + while (true) { + auto is_done = harness.ServeConformanceRequest(); + if (!is_done.ok()) { + GOOGLE_LOG(FATAL) << is_done.status(); + } + if (*is_done) { + break; } + total_runs++; } + GOOGLE_LOG(INFO) << "conformance-cpp: recieved EOF from test runner after " + << total_runs << " tests"; } diff --git a/csharp/.gitignore b/csharp/.gitignore index d0d7ae04f7..62c96383ce 100644 --- a/csharp/.gitignore +++ b/csharp/.gitignore @@ -18,6 +18,7 @@ lib/NUnit # Untracked files # .vs +.cr *.user *.suo *.nupkg diff --git a/csharp/src/Google.Protobuf.Test/ByteStringTest.cs b/csharp/src/Google.Protobuf.Test/ByteStringTest.cs index 4876e09798..3810565faa 100644 --- a/csharp/src/Google.Protobuf.Test/ByteStringTest.cs +++ b/csharp/src/Google.Protobuf.Test/ByteStringTest.cs @@ -41,9 +41,7 @@ using System.Buffers; using System.Runtime.InteropServices; using System.Threading; using System.Runtime.CompilerServices; -#if !NET35 using System.Threading.Tasks; -#endif namespace Google.Protobuf { @@ -349,7 +347,6 @@ namespace Google.Protobuf Assert.AreEqual(expected, actual, $"{expected.ToBase64()} != {actual.ToBase64()}"); } -#if !NET35 [Test] public async Task FromStreamAsync_Seekable() { @@ -373,7 +370,6 @@ namespace Google.Protobuf ByteString expected = ByteString.CopyFrom(2, 3, 4); Assert.AreEqual(expected, actual, $"{expected.ToBase64()} != {actual.ToBase64()}"); } -#endif [Test] public void GetHashCode_Regression() diff --git a/csharp/src/Google.Protobuf.Test/Collections/MapFieldTest.cs b/csharp/src/Google.Protobuf.Test/Collections/MapFieldTest.cs index 1f7b8a35cf..abd19a2d8e 100644 --- a/csharp/src/Google.Protobuf.Test/Collections/MapFieldTest.cs +++ b/csharp/src/Google.Protobuf.Test/Collections/MapFieldTest.cs @@ -635,7 +635,6 @@ namespace Google.Protobuf.Collections Assert.IsTrue(input.IsAtEnd); } -#if !NET35 [Test] public void IDictionaryKeys_Equals_IReadOnlyDictionaryKeys() { @@ -649,7 +648,6 @@ namespace Google.Protobuf.Collections var map = new MapField { { "foo", "bar" }, { "x", "y" } }; CollectionAssert.AreEquivalent(((IDictionary)map).Values, ((IReadOnlyDictionary)map).Values); } -#endif private static KeyValuePair NewKeyValuePair(TKey key, TValue value) { diff --git a/csharp/src/Google.Protobuf.Test/Compatibility/StreamExtensionsTest.cs b/csharp/src/Google.Protobuf.Test/Compatibility/StreamExtensionsTest.cs deleted file mode 100644 index 48c0725fa6..0000000000 --- a/csharp/src/Google.Protobuf.Test/Compatibility/StreamExtensionsTest.cs +++ /dev/null @@ -1,67 +0,0 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2015 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -#if NET35 -using System; -using System.IO; -using NUnit.Framework; -using Google.Protobuf.Compatibility; - -namespace Google.Protobuf.Test.Compatibility -{ - public class StreamExtensionsTest - { - [Test] - public void CopyToNullArgument() - { - var memoryStream = new MemoryStream(); - Assert.Throws(() => memoryStream.CopyTo(null)); - } - - [Test] - public void CopyToTest() - { - byte[] bytesToStream = new byte[] { 0x31, 0x08, 0xFF, 0x00 }; - Stream source = new MemoryStream(bytesToStream); - Stream destination = new MemoryStream((int)source.Length); - source.CopyTo(destination); - destination.Seek(0, SeekOrigin.Begin); - - Assert.AreEqual(0x31, destination.ReadByte()); - Assert.AreEqual(0x08, destination.ReadByte()); - Assert.AreEqual(0xFF, destination.ReadByte()); - Assert.AreEqual(0x00, destination.ReadByte()); - Assert.AreEqual(-1, destination.ReadByte()); - } - } -} -#endif diff --git a/csharp/src/Google.Protobuf.Test/Compatibility/TypeExtensionsTest.cs b/csharp/src/Google.Protobuf.Test/Compatibility/TypeExtensionsTest.cs index 5d86c20896..1d69fcb2be 100644 --- a/csharp/src/Google.Protobuf.Test/Compatibility/TypeExtensionsTest.cs +++ b/csharp/src/Google.Protobuf.Test/Compatibility/TypeExtensionsTest.cs @@ -34,7 +34,6 @@ using System; using System.Collections.Generic; using System.Reflection; -#if !NET35 namespace Google.Protobuf.Compatibility { public class TypeExtensionsTest @@ -114,4 +113,3 @@ namespace Google.Protobuf.Compatibility } } } -#endif diff --git a/csharp/src/Google.Protobuf.Test/FieldCodecTest.cs b/csharp/src/Google.Protobuf.Test/FieldCodecTest.cs index 7f366926fa..4f2a5625c7 100644 --- a/csharp/src/Google.Protobuf.Test/FieldCodecTest.cs +++ b/csharp/src/Google.Protobuf.Test/FieldCodecTest.cs @@ -169,7 +169,6 @@ namespace Google.Protobuf // WriteTagAndValue ignores default values var stream = new MemoryStream(); CodedOutputStream codedOutput; -#if !NET35 codedOutput = new CodedOutputStream(stream); codec.WriteTagAndValue(codedOutput, codec.DefaultValue); codedOutput.Flush(); @@ -179,7 +178,6 @@ namespace Google.Protobuf { Assert.AreEqual(default(T), codec.DefaultValue); } -#endif // The plain ValueWriter/ValueReader delegates don't. if (codec.DefaultValue != null) // This part isn't appropriate for message types. diff --git a/csharp/src/Google.Protobuf/ByteString.cs b/csharp/src/Google.Protobuf/ByteString.cs index 8c6eb5b3be..c615616360 100644 --- a/csharp/src/Google.Protobuf/ByteString.cs +++ b/csharp/src/Google.Protobuf/ByteString.cs @@ -37,13 +37,8 @@ using System.IO; using System.Runtime.InteropServices; using System.Security; using System.Text; -#if !NET35 using System.Threading; using System.Threading.Tasks; -#endif -#if NET35 -using Google.Protobuf.Compatibility; -#endif namespace Google.Protobuf { @@ -186,7 +181,6 @@ namespace Google.Protobuf return AttachBytes(bytes); } -#if !NET35 /// /// Constructs a from data in the given stream, asynchronously. /// @@ -200,7 +194,6 @@ namespace Google.Protobuf ProtoPreconditions.CheckNotNull(stream, nameof(stream)); return ByteStringAsync.FromStreamAsyncCore(stream, cancellationToken); } -#endif /// /// Constructs a from the given array. The contents diff --git a/csharp/src/Google.Protobuf/ByteStringAsync.cs b/csharp/src/Google.Protobuf/ByteStringAsync.cs index 3465cc67b4..5aa92f026d 100644 --- a/csharp/src/Google.Protobuf/ByteStringAsync.cs +++ b/csharp/src/Google.Protobuf/ByteStringAsync.cs @@ -43,7 +43,6 @@ namespace Google.Protobuf /// internal static class ByteStringAsync { -#if !NET35 internal static async Task FromStreamAsyncCore(Stream stream, CancellationToken cancellationToken) { int capacity = stream.CanSeek ? checked((int)(stream.Length - stream.Position)) : 0; @@ -59,6 +58,5 @@ namespace Google.Protobuf #endif return ByteString.AttachBytes(bytes); } -#endif } } \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/Collections/MapField.cs b/csharp/src/Google.Protobuf/Collections/MapField.cs index e5217f4514..0f02d59454 100644 --- a/csharp/src/Google.Protobuf/Collections/MapField.cs +++ b/csharp/src/Google.Protobuf/Collections/MapField.cs @@ -68,10 +68,7 @@ namespace Google.Protobuf.Collections /// in future versions. /// /// - public sealed class MapField : IDeepCloneable>, IDictionary, IEquatable>, IDictionary -#if !NET35 - , IReadOnlyDictionary -#endif + public sealed class MapField : IDeepCloneable>, IDictionary, IEquatable>, IDictionary, IReadOnlyDictionary { private static readonly EqualityComparer ValueEqualityComparer = ProtobufEqualityComparers.GetEqualityComparer(); private static readonly EqualityComparer KeyEqualityComparer = ProtobufEqualityComparers.GetEqualityComparer(); @@ -600,11 +597,8 @@ namespace Google.Protobuf.Collections #endregion #region IReadOnlyDictionary explicit interface implementation -#if !NET35 IEnumerable IReadOnlyDictionary.Keys => Keys; - IEnumerable IReadOnlyDictionary.Values => Values; -#endif #endregion private class DictionaryEnumerator : IDictionaryEnumerator diff --git a/csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs b/csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs deleted file mode 100644 index 28530a29a8..0000000000 --- a/csharp/src/Google.Protobuf/Collections/ReadOnlyDictionary.cs +++ /dev/null @@ -1,147 +0,0 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -using System; -using System.Collections; -using System.Collections.Generic; - -namespace Google.Protobuf.Collections -{ - /// - /// Read-only wrapper around another dictionary. - /// - internal sealed class ReadOnlyDictionary : IDictionary - { - private readonly IDictionary wrapped; - - public ReadOnlyDictionary(IDictionary wrapped) - { - this.wrapped = wrapped; - } - - public void Add(TKey key, TValue value) - { - throw new InvalidOperationException(); - } - - public bool ContainsKey(TKey key) - { - return wrapped.ContainsKey(key); - } - - public ICollection Keys - { - get { return wrapped.Keys; } - } - - public bool Remove(TKey key) - { - throw new InvalidOperationException(); - } - - public bool TryGetValue(TKey key, out TValue value) - { - return wrapped.TryGetValue(key, out value); - } - - public ICollection Values - { - get { return wrapped.Values; } - } - - public TValue this[TKey key] - { - get { return wrapped[key]; } - set { throw new InvalidOperationException(); } - } - - public void Add(KeyValuePair item) - { - throw new InvalidOperationException(); - } - - public void Clear() - { - throw new InvalidOperationException(); - } - - public bool Contains(KeyValuePair item) - { - return wrapped.Contains(item); - } - - public void CopyTo(KeyValuePair[] array, int arrayIndex) - { - wrapped.CopyTo(array, arrayIndex); - } - - public int Count - { - get { return wrapped.Count; } - } - - public bool IsReadOnly - { - get { return true; } - } - - public bool Remove(KeyValuePair item) - { - throw new InvalidOperationException(); - } - - public IEnumerator> GetEnumerator() - { - return wrapped.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return ((IEnumerable) wrapped).GetEnumerator(); - } - - public override bool Equals(object obj) - { - return wrapped.Equals(obj); - } - - public override int GetHashCode() - { - return wrapped.GetHashCode(); - } - - public override string ToString() - { - return wrapped.ToString(); - } - } -} \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/Collections/RepeatedField.cs b/csharp/src/Google.Protobuf/Collections/RepeatedField.cs index 9269c7444c..cd6f5eb572 100644 --- a/csharp/src/Google.Protobuf/Collections/RepeatedField.cs +++ b/csharp/src/Google.Protobuf/Collections/RepeatedField.cs @@ -48,10 +48,7 @@ namespace Google.Protobuf.Collections /// supported by Protocol Buffers but nor does it guarantee that all operations will work in such cases. /// /// The element type of the repeated field. - public sealed class RepeatedField : IList, IList, IDeepCloneable>, IEquatable> -#if !NET35 - , IReadOnlyList -#endif + public sealed class RepeatedField : IList, IList, IDeepCloneable>, IEquatable>, IReadOnlyList { private static readonly EqualityComparer EqualityComparer = ProtobufEqualityComparers.GetEqualityComparer(); private static readonly T[] EmptyArray = new T[0]; diff --git a/csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs b/csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs index 95a02c727f..8a6fefa74e 100644 --- a/csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs +++ b/csharp/src/Google.Protobuf/Compatibility/PropertyInfoExtensions.cs @@ -47,11 +47,7 @@ namespace Google.Protobuf.Compatibility /// internal static MethodInfo GetGetMethod(this PropertyInfo target) { -#if NET35 - var method = target.GetGetMethod(); -#else var method = target.GetMethod; -#endif return method != null && method.IsPublic ? method : null; } @@ -61,11 +57,7 @@ namespace Google.Protobuf.Compatibility /// internal static MethodInfo GetSetMethod(this PropertyInfo target) { -#if NET35 - var method = target.GetSetMethod(); -#else var method = target.SetMethod; -#endif return method != null && method.IsPublic ? method : null; } } diff --git a/csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs b/csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs deleted file mode 100644 index bf4bf22018..0000000000 --- a/csharp/src/Google.Protobuf/Compatibility/StreamExtensions.cs +++ /dev/null @@ -1,66 +0,0 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2015 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -#if NET35 -using System; -using System.IO; - -namespace Google.Protobuf.Compatibility -{ - /// - /// Extension methods for in order to provide - /// backwards compatibility with .NET 3.5 - /// - public static class StreamExtensions - { - // 81920 seems to be the default buffer size used in .NET 4.5.1 - private const int BUFFER_SIZE = 81920; - - /// - /// Write the contents of the current stream to the destination stream - /// - public static void CopyTo(this Stream source, Stream destination) - { - if (destination == null) - { - throw new ArgumentNullException(nameof(destination)); - } - - byte[] buffer = new byte[BUFFER_SIZE]; - int numBytesRead; - while ((numBytesRead = source.Read(buffer, 0, buffer.Length)) > 0) { - destination.Write(buffer, 0, numBytesRead); - } - } - } -} -#endif diff --git a/csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs b/csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs index b3acda2da7..5db5dca9c3 100644 --- a/csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs +++ b/csharp/src/Google.Protobuf/Compatibility/TypeExtensions.cs @@ -34,7 +34,6 @@ using System; using System.Diagnostics.CodeAnalysis; using System.Reflection; -#if !NET35 namespace Google.Protobuf.Compatibility { /// @@ -112,4 +111,3 @@ namespace Google.Protobuf.Compatibility } } } -#endif diff --git a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs index 40a6ff832b..415b7bec83 100644 --- a/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/MessageDescriptor.cs @@ -36,10 +36,6 @@ using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; -#if NET35 -// Needed for ReadOnlyDictionary, which does not exist in .NET 3.5 -using Google.Protobuf.Collections; -#endif namespace Google.Protobuf.Reflection { diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMaskPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMaskPartial.cs index 58a33cb6d4..dd00e2d78c 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMaskPartial.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMaskPartial.cs @@ -63,12 +63,7 @@ namespace Google.Protobuf.WellKnownTypes if (firstInvalid == null) { var writer = new StringWriter(); -#if NET35 - var query = paths.Select(JsonFormatter.ToJsonName); - JsonFormatter.WriteString(writer, string.Join(",", query.ToArray())); -#else JsonFormatter.WriteString(writer, string.Join(",", paths.Select(JsonFormatter.ToJsonName))); -#endif return writer.ToString(); } else diff --git a/docs/options.md b/docs/options.md index 8c22af12b0..62a32d37d5 100644 --- a/docs/options.md +++ b/docs/options.md @@ -328,3 +328,7 @@ with info about your project (name and website) so we can add an entry for you. 1. Protoc-gen-xo * Website: https://github.com/xo/ecosystem * Extension: 1147 + +1. Ballerina gRPC + * Website: https://github.com/ballerina-platform/module-ballerina-grpc + * Extension: 1148 diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index f1e5ac7939..2b8f807951 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -69,15 +69,13 @@ import java.util.Map; import java.util.TreeMap; /** - * All generated protocol message classes extend this class. This class - * implements most of the Message and Builder interfaces using Java reflection. - * Users can ignore this class and pretend that generated messages implement - * the Message interface directly. + * All generated protocol message classes extend this class. This class implements most of the + * Message and Builder interfaces using Java reflection. Users can ignore this class and pretend + * that generated messages implement the Message interface directly. * * @author kenton@google.com Kenton Varda */ -public abstract class GeneratedMessageV3 extends AbstractMessage - implements Serializable { +public abstract class GeneratedMessageV3 extends AbstractMessage implements Serializable { private static final long serialVersionUID = 1L; /** @@ -87,7 +85,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage */ protected static boolean alwaysUseFieldBuilders = false; - /** For use by generated code only. */ + /** For use by generated code only. */ protected UnknownFieldSet unknownFields; protected GeneratedMessageV3() { @@ -100,31 +98,29 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Parser getParserForType() { - throw new UnsupportedOperationException( - "This is supposed to be overridden by subclasses."); + throw new UnsupportedOperationException("This is supposed to be overridden by subclasses."); } - /** - * @see #setAlwaysUseFieldBuildersForTesting(boolean) - */ + /** + * @see #setAlwaysUseFieldBuildersForTesting(boolean) + */ static void enableAlwaysUseFieldBuildersForTesting() { setAlwaysUseFieldBuildersForTesting(true); } /** - * For testing. Allows a test to disable/re-enable the optimization that avoids - * using field builders for nested messages until they are requested. By disabling - * this optimization, existing tests can be reused to test the field builders. - * See {@link RepeatedFieldBuilder} and {@link SingleFieldBuilder}. + * For testing. Allows a test to disable/re-enable the optimization that avoids using field + * builders for nested messages until they are requested. By disabling this optimization, existing + * tests can be reused to test the field builders. See {@link RepeatedFieldBuilder} and {@link + * SingleFieldBuilder}. */ static void setAlwaysUseFieldBuildersForTesting(boolean useBuilders) { alwaysUseFieldBuilders = useBuilders; } /** - * Get the FieldAccessorTable for this type. We can't have the message - * class pass this in to the constructor because of bootstrapping trouble - * with DescriptorProtos. + * Get the FieldAccessorTable for this type. We can't have the message class pass this in to the + * constructor because of bootstrapping trouble with DescriptorProtos. */ protected abstract FieldAccessorTable internalGetFieldAccessorTable(); @@ -149,16 +145,14 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Internal helper to return a modifiable map containing all the fields. - * The returned Map is modifiable so that the caller can add additional - * extension fields to implement {@link #getAllFields()}. + * Internal helper to return a modifiable map containing all the fields. The returned Map is + * modifiable so that the caller can add additional extension fields to implement {@link + * #getAllFields()}. * * @param getBytesForString whether to generate ByteString for string fields */ - private Map getAllFieldsMutable( - boolean getBytesForString) { - final TreeMap result = - new TreeMap(); + private Map getAllFieldsMutable(boolean getBytesForString) { + final TreeMap result = new TreeMap(); final Descriptor descriptor = internalGetFieldAccessorTable().descriptor; final List fields = descriptor.getFields(); @@ -214,8 +208,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // Check that embedded messages are initialized. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isRepeated()) { - @SuppressWarnings("unchecked") final - List messageList = (List) getField(field); + @SuppressWarnings("unchecked") + final List messageList = (List) getField(field); for (final Message element : messageList) { if (!element.isInitialized()) { return false; @@ -234,23 +228,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Map getAllFields() { - return Collections.unmodifiableMap( - getAllFieldsMutable(/* getBytesForString = */ false)); + return Collections.unmodifiableMap(getAllFieldsMutable(/* getBytesForString = */ false)); } /** - * Returns a collection of all the fields in this message which are set - * and their corresponding values. A singular ("required" or "optional") - * field is set iff hasField() returns true for that field. A "repeated" - * field is set iff getRepeatedFieldCount() is greater than zero. The - * values are exactly what would be returned by calling - * {@link #getFieldRaw(Descriptors.FieldDescriptor)} for each field. The map - * is guaranteed to be a sorted map, so iterating over it will return fields - * in order by field number. + * Returns a collection of all the fields in this message which are set and their corresponding + * values. A singular ("required" or "optional") field is set iff hasField() returns true for that + * field. A "repeated" field is set iff getRepeatedFieldCount() is greater than zero. The values + * are exactly what would be returned by calling {@link #getFieldRaw(Descriptors.FieldDescriptor)} + * for each field. The map is guaranteed to be a sorted map, so iterating over it will return + * fields in order by field number. */ Map getAllFieldsRaw() { - return Collections.unmodifiableMap( - getAllFieldsMutable(/* getBytesForString = */ true)); + return Collections.unmodifiableMap(getAllFieldsMutable(/* getBytesForString = */ true)); } @Override @@ -274,12 +264,11 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Obtains the value of the given field, or the default value if it is - * not set. For primitive fields, the boxed primitive value is returned. - * For enum fields, the EnumValueDescriptor for the value is returned. For - * embedded message fields, the sub-message is returned. For repeated - * fields, a java.util.List is returned. For present string fields, a - * ByteString is returned representing the bytes that the field contains. + * Obtains the value of the given field, or the default value if it is not set. For primitive + * fields, the boxed primitive value is returned. For enum fields, the EnumValueDescriptor for the + * value is returned. For embedded message fields, the sub-message is returned. For repeated + * fields, a java.util.List is returned. For present string fields, a ByteString is returned + * representing the bytes that the field contains. */ Object getFieldRaw(final FieldDescriptor field) { return internalGetFieldAccessorTable().getField(field).getRaw(this); @@ -287,20 +276,17 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public int getRepeatedFieldCount(final FieldDescriptor field) { - return internalGetFieldAccessorTable().getField(field) - .getRepeatedCount(this); + return internalGetFieldAccessorTable().getField(field).getRepeatedCount(this); } @Override public Object getRepeatedField(final FieldDescriptor field, final int index) { - return internalGetFieldAccessorTable().getField(field) - .getRepeated(this, index); + return internalGetFieldAccessorTable().getField(field).getRepeated(this, index); } @Override public UnknownFieldSet getUnknownFields() { - throw new UnsupportedOperationException( - "This is supposed to be overridden by subclasses."); + throw new UnsupportedOperationException("This is supposed to be overridden by subclasses."); } /** @@ -342,8 +328,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, InputStream input, - ExtensionRegistryLite extensions) throws IOException { + protected static M parseWithIOException( + Parser parser, InputStream input, ExtensionRegistryLite extensions) throws IOException { try { return parser.parseFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -351,8 +337,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, - CodedInputStream input) throws IOException { + protected static M parseWithIOException( + Parser parser, CodedInputStream input) throws IOException { try { return parser.parseFrom(input); } catch (InvalidProtocolBufferException e) { @@ -360,8 +346,9 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, - CodedInputStream input, ExtensionRegistryLite extensions) throws IOException { + protected static M parseWithIOException( + Parser parser, CodedInputStream input, ExtensionRegistryLite extensions) + throws IOException { try { return parser.parseFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -369,8 +356,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseDelimitedWithIOException(Parser parser, - InputStream input) throws IOException { + protected static M parseDelimitedWithIOException( + Parser parser, InputStream input) throws IOException { try { return parser.parseDelimitedFrom(input); } catch (InvalidProtocolBufferException e) { @@ -378,8 +365,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseDelimitedWithIOException(Parser parser, - InputStream input, ExtensionRegistryLite extensions) throws IOException { + protected static M parseDelimitedWithIOException( + Parser parser, InputStream input, ExtensionRegistryLite extensions) throws IOException { try { return parser.parseDelimitedFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -480,63 +467,53 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } - /** * This class is used to make a generated protected method inaccessible from user's code (e.g., * the {@link #newInstance} method below). When this class is used as a parameter's type in a - * generated protected method, the method is visible to user's code in the same package, but - * since the constructor of this class is private to protobuf runtime, user's code can't obtain - * an instance of this class and as such can't actually make a method call on the protected - * method. + * generated protected method, the method is visible to user's code in the same package, but since + * the constructor of this class is private to protobuf runtime, user's code can't obtain an + * instance of this class and as such can't actually make a method call on the protected method. */ protected static final class UnusedPrivateParameter { static final UnusedPrivateParameter INSTANCE = new UnusedPrivateParameter(); - private UnusedPrivateParameter() { - } + private UnusedPrivateParameter() {} } - /** - * Creates a new instance of this message type. Overridden in the generated code. - */ + /** Creates a new instance of this message type. Overridden in the generated code. */ @SuppressWarnings({"unused"}) protected Object newInstance(UnusedPrivateParameter unused) { throw new UnsupportedOperationException("This method must be overridden by the subclass."); } - /** - * Used by parsing constructors in generated classes. - */ + /** Used by parsing constructors in generated classes. */ protected void makeExtensionsImmutable() { // Noop for messages without extensions. } /** - * TODO(xiaofeng): remove this after b/29368482 is fixed. We need to move this - * interface to AbstractMessage in order to versioning GeneratedMessageV3 but - * this move breaks binary compatibility for AppEngine. After AppEngine is - * fixed we can exclude this from google3. + * TODO(xiaofeng): remove this after b/29368482 is fixed. We need to move this interface to + * AbstractMessage in order to versioning GeneratedMessageV3 but this move breaks binary + * compatibility for AppEngine. After AppEngine is fixed we can exclude this from google3. */ protected interface BuilderParent extends AbstractMessage.BuilderParent {} - /** - * TODO(xiaofeng): remove this together with GeneratedMessageV3.BuilderParent. - */ + /** TODO(xiaofeng): remove this together with GeneratedMessageV3.BuilderParent. */ protected abstract Message.Builder newBuilderForType(BuilderParent parent); @Override protected Message.Builder newBuilderForType(final AbstractMessage.BuilderParent parent) { - return newBuilderForType(new BuilderParent() { - @Override - public void markDirty() { - parent.markDirty(); - } - }); + return newBuilderForType( + new BuilderParent() { + @Override + public void markDirty() { + parent.markDirty(); + } + }); } - @SuppressWarnings("unchecked") - public abstract static class Builder > + public abstract static class Builder> extends AbstractMessage.Builder { private BuilderParent builderParent; @@ -547,8 +524,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // to dispatch dirty invalidations. See GeneratedMessageV3.BuilderListener. private boolean isClean; - private UnknownFieldSet unknownFields = - UnknownFieldSet.getDefaultInstance(); + private UnknownFieldSet unknownFields = UnknownFieldSet.getDefaultInstance(); protected Builder() { this(null); @@ -563,9 +539,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage builderParent = null; } - /** - * Called by the subclass when a message is built. - */ + /** Called by the subclass when a message is built. */ protected void onBuilt() { if (builderParent != null) { markClean(); @@ -573,8 +547,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called by the subclass or a builder to notify us that a message was - * built and may be cached and therefore invalidations are needed. + * Called by the subclass or a builder to notify us that a message was built and may be cached + * and therefore invalidations are needed. */ @Override protected void markClean() { @@ -592,15 +566,14 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public BuilderType clone() { - BuilderType builder = - (BuilderType) getDefaultInstanceForType().newBuilderForType(); + BuilderType builder = (BuilderType) getDefaultInstanceForType().newBuilderForType(); builder.mergeFrom(buildPartial()); return builder; } /** - * Called by the initialization and clear code paths to allow subclasses to - * reset any of their builtin fields back to the initial values. + * Called by the initialization and clear code paths to allow subclasses to reset any of their + * builtin fields back to the initial values. */ @Override public BuilderType clear() { @@ -610,9 +583,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Get the FieldAccessorTable for this type. We can't have the message - * class pass this in to the constructor because of bootstrapping trouble - * with DescriptorProtos. + * Get the FieldAccessorTable for this type. We can't have the message class pass this in to the + * constructor because of bootstrapping trouble with DescriptorProtos. */ protected abstract FieldAccessorTable internalGetFieldAccessorTable(); @@ -628,8 +600,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Internal helper which returns a mutable map. */ private Map getAllFieldsMutable() { - final TreeMap result = - new TreeMap(); + final TreeMap result = new TreeMap(); final Descriptor descriptor = internalGetFieldAccessorTable().descriptor; final List fields = descriptor.getFields(); @@ -681,8 +652,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Message.Builder getRepeatedFieldBuilder(final FieldDescriptor field, int index) { - return internalGetFieldAccessorTable().getField(field).getRepeatedBuilder( - this, index); + return internalGetFieldAccessorTable().getField(field).getRepeatedBuilder(this, index); } @Override @@ -732,21 +702,18 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public int getRepeatedFieldCount(final FieldDescriptor field) { - return internalGetFieldAccessorTable().getField(field) - .getRepeatedCount(this); + return internalGetFieldAccessorTable().getField(field).getRepeatedCount(this); } @Override public Object getRepeatedField(final FieldDescriptor field, final int index) { - return internalGetFieldAccessorTable().getField(field) - .getRepeated(this, index); + return internalGetFieldAccessorTable().getField(field).getRepeated(this, index); } @Override public BuilderType setRepeatedField( final FieldDescriptor field, final int index, final Object value) { - internalGetFieldAccessorTable().getField(field) - .setRepeated(this, index, value); + internalGetFieldAccessorTable().getField(field).setRepeated(this, index, value); return (BuilderType) this; } @@ -768,20 +735,16 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * This method is obsolete, but we must retain it for compatibility with - * older generated code. + * This method is obsolete, but we must retain it for compatibility with older generated code. */ protected BuilderType setUnknownFieldsProto3(final UnknownFieldSet unknownFields) { return setUnknownFieldsInternal(unknownFields); } @Override - public BuilderType mergeUnknownFields( - final UnknownFieldSet unknownFields) { + public BuilderType mergeUnknownFields(final UnknownFieldSet unknownFields) { return setUnknownFields( - UnknownFieldSet.newBuilder(this.unknownFields) - .mergeFrom(unknownFields) - .build()); + UnknownFieldSet.newBuilder(this.unknownFields).mergeFrom(unknownFields).build()); } @@ -797,16 +760,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // Check that embedded messages are initialized. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isRepeated()) { - @SuppressWarnings("unchecked") final - List messageList = (List) getField(field); + @SuppressWarnings("unchecked") + final List messageList = (List) getField(field); for (final Message element : messageList) { if (!element.isInitialized()) { return false; } } } else { - if (hasField(field) && - !((Message) getField(field)).isInitialized()) { + if (hasField(field) && !((Message) getField(field)).isInitialized()) { return false; } } @@ -821,9 +783,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Implementation of {@link BuilderParent} for giving to our children. This - * small inner class makes it so we don't publicly expose the BuilderParent - * methods. + * Implementation of {@link BuilderParent} for giving to our children. This small inner class + * makes it so we don't publicly expose the BuilderParent methods. */ private class BuilderParentImpl implements BuilderParent { @@ -835,6 +796,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Gets the {@link BuilderParent} for giving to our children. + * * @return The builder parent for our children. */ protected BuilderParent getParentForChildren() { @@ -845,8 +807,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called when a the builder or one of its nested children has changed - * and any parent should be notified of its invalidation. + * Called when a the builder or one of its nested children has changed and any parent should be + * notified of its invalidation. */ protected final void onChanged() { if (isClean && builderParent != null) { @@ -858,22 +820,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Gets the map field with the given field number. This method should be - * overridden in the generated message class if the message contains map - * fields. + * Gets the map field with the given field number. This method should be overridden in the + * generated message class if the message contains map fields. * - * Unlike other field types, reflection support for map fields can't be - * implemented based on generated public API because we need to access a - * map field as a list in reflection API but the generated API only allows - * us to access it as a map. This method returns the underlying map field - * directly and thus enables us to access the map field as a list. + *

Unlike other field types, reflection support for map fields can't be implemented based on + * generated public API because we need to access a map field as a list in reflection API but + * the generated API only allows us to access it as a map. This method returns the underlying + * map field directly and thus enables us to access the map field as a list. */ @SuppressWarnings({"unused", "rawtypes"}) protected MapField internalGetMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } /** Like {@link #internalGetMapField} but return a mutable version. */ @@ -881,36 +840,30 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected MapField internalGetMutableMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } } // ================================================================= // Extensions-related stuff - public interface ExtendableMessageOrBuilder< - MessageType extends ExtendableMessage> extends MessageOrBuilder { + public interface ExtendableMessageOrBuilder + extends MessageOrBuilder { // Re-define for return type covariance. @Override Message getDefaultInstanceForType(); /** Check if a singular extension is present. */ - boolean hasExtension( - ExtensionLite extension); + boolean hasExtension(ExtensionLite extension); /** Get the number of elements in a repeated extension. */ - int getExtensionCount( - ExtensionLite> extension); + int getExtensionCount(ExtensionLite> extension); /** Get the value of an extension. */ - Type getExtension( - ExtensionLite extension); + Type getExtension(ExtensionLite extension); /** Get one element of a repeated extension. */ - Type getExtension( - ExtensionLite> extension, - int index); + Type getExtension(ExtensionLite> extension, int index); /** Check if a singular extension is present. */ boolean hasExtension( @@ -941,16 +894,13 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Generated message classes for message types that contain extension ranges - * subclass this. + * Generated message classes for message types that contain extension ranges subclass this. * - *

This class implements type-safe accessors for extensions. They - * implement all the same operations that you can do with normal fields -- - * e.g. "has", "get", and "getCount" -- but for extensions. The extensions - * are identified using instances of the class {@link GeneratedExtension}; - * the protocol compiler generates a static instance of this class for every - * extension in its input. Through the magic of generics, all is made - * type-safe. + *

This class implements type-safe accessors for extensions. They implement all the same + * operations that you can do with normal fields -- e.g. "has", "get", and "getCount" -- but for + * extensions. The extensions are identified using instances of the class {@link + * GeneratedExtension}; the protocol compiler generates a static instance of this class for every + * extension in its input. Through the magic of generics, all is made type-safe. * *

For example, imagine you have the {@code .proto} file: * @@ -975,10 +925,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage * *

See also {@link ExtendableBuilder}. */ - public abstract static class ExtendableMessage< - MessageType extends ExtendableMessage> - extends GeneratedMessageV3 - implements ExtendableMessageOrBuilder { + public abstract static class ExtendableMessage + extends GeneratedMessageV3 implements ExtendableMessageOrBuilder { private static final long serialVersionUID = 1L; @@ -988,22 +936,20 @@ public abstract class GeneratedMessageV3 extends AbstractMessage this.extensions = FieldSet.newFieldSet(); } - protected ExtendableMessage( - ExtendableBuilder builder) { + protected ExtendableMessage(ExtendableBuilder builder) { super(builder); this.extensions = builder.buildExtensions(); } - private void verifyExtensionContainingType( - final Extension extension) { - if (extension.getDescriptor().getContainingType() != - getDescriptorForType()) { + private void verifyExtensionContainingType(final Extension extension) { + if (extension.getDescriptor().getContainingType() != getDescriptorForType()) { // This can only happen if someone uses unchecked operations. throw new IllegalArgumentException( - "Extension is for type \"" + - extension.getDescriptor().getContainingType().getFullName() + - "\" which does not match message type \"" + - getDescriptorForType().getFullName() + "\"."); + "Extension is for type \"" + + extension.getDescriptor().getContainingType().getFullName() + + "\" which does not match message type \"" + + getDescriptorForType().getFullName() + + "\"."); } } @@ -1041,12 +987,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (value == null) { if (descriptor.isRepeated()) { return (Type) Collections.emptyList(); - } else if (descriptor.getJavaType() == - FieldDescriptor.JavaType.MESSAGE) { + } else if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { return (Type) extension.getMessageDefaultInstance(); } else { - return (Type) extension.fromReflectionType( - descriptor.getDefaultValue()); + return (Type) extension.fromReflectionType(descriptor.getDefaultValue()); } } else { return (Type) extension.fromReflectionType(value); @@ -1062,8 +1006,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage verifyExtensionContainingType(extension); FieldDescriptor descriptor = extension.getDescriptor(); - return (Type) extension.singularFromReflectionType( - extensions.getRepeatedField(descriptor, index)); + return (Type) + extension.singularFromReflectionType(extensions.getRepeatedField(descriptor, index)); } /** Check if a singular extension is present. */ @@ -1128,10 +1072,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedInputStream input, UnknownFieldSet.Builder unknownFields, ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + int tag) + throws IOException { return MessageReflection.mergeFieldFrom( - input, input.shouldDiscardUnknownFields() ? null : unknownFields, extensionRegistry, - getDescriptorForType(), new MessageReflection.ExtensionAdapter(extensions), tag); + input, + input.shouldDiscardUnknownFields() ? null : unknownFields, + extensionRegistry, + getDescriptorForType(), + new MessageReflection.ExtensionAdapter(extensions), + tag); } /** @@ -1143,31 +1092,28 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedInputStream input, UnknownFieldSet.Builder unknownFields, ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + int tag) + throws IOException { return parseUnknownField(input, unknownFields, extensionRegistry, tag); } - /** - * Used by parsing constructors in generated classes. - */ + /** Used by parsing constructors in generated classes. */ @Override protected void makeExtensionsImmutable() { extensions.makeImmutable(); } /** - * Used by subclasses to serialize extensions. Extension ranges may be - * interleaved with field numbers, but we must write them in canonical - * (sorted by field number) order. ExtensionWriter helps us write - * individual ranges of extensions at once. + * Used by subclasses to serialize extensions. Extension ranges may be interleaved with field + * numbers, but we must write them in canonical (sorted by field number) order. ExtensionWriter + * helps us write individual ranges of extensions at once. */ protected class ExtensionWriter { // Imagine how much simpler this code would be if Java iterators had // a way to get the next element without advancing the iterator. - private final Iterator> iter = - extensions.iterator(); + private final Iterator> iter = extensions.iterator(); private Map.Entry next; private final boolean messageSetWireFormat; @@ -1178,19 +1124,18 @@ public abstract class GeneratedMessageV3 extends AbstractMessage this.messageSetWireFormat = messageSetWireFormat; } - public void writeUntil(final int end, final CodedOutputStream output) - throws IOException { + public void writeUntil(final int end, final CodedOutputStream output) throws IOException { while (next != null && next.getKey().getNumber() < end) { FieldDescriptor descriptor = next.getKey(); - if (messageSetWireFormat && descriptor.getLiteJavaType() == - WireFormat.JavaType.MESSAGE && - !descriptor.isRepeated()) { + if (messageSetWireFormat + && descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE + && !descriptor.isRepeated()) { if (next instanceof LazyField.LazyEntry) { - output.writeRawMessageSetExtension(descriptor.getNumber(), + output.writeRawMessageSetExtension( + descriptor.getNumber(), ((LazyField.LazyEntry) next).getField().toByteString()); } else { - output.writeMessageSetExtension(descriptor.getNumber(), - (Message) next.getValue()); + output.writeMessageSetExtension(descriptor.getNumber(), (Message) next.getValue()); } } else { // TODO(xiangl): Taken care of following code, it may cause @@ -1214,6 +1159,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected ExtensionWriter newExtensionWriter() { return new ExtensionWriter(false); } + protected ExtensionWriter newMessageSetExtensionWriter() { return new ExtensionWriter(true); } @@ -1222,6 +1168,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected int extensionsSerializedSize() { return extensions.getSerializedSize(); } + protected int extensionsSerializedSizeAsMessageSet() { return extensions.getMessageSetSerializedSize(); } @@ -1293,8 +1240,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeatedField(final FieldDescriptor field, - final int index) { + public Object getRepeatedField(final FieldDescriptor field, final int index) { if (field.isExtension()) { verifyContainingType(field); return extensions.getRepeatedField(field, index); @@ -1305,23 +1251,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private void verifyContainingType(final FieldDescriptor field) { if (field.getContainingType() != getDescriptorForType()) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } } } /** - * Generated message builders for message types that contain extension ranges - * subclass this. + * Generated message builders for message types that contain extension ranges subclass this. * - *

This class implements type-safe accessors for extensions. They - * implement all the same operations that you can do with normal fields -- - * e.g. "get", "set", and "add" -- but for extensions. The extensions are - * identified using instances of the class {@link GeneratedExtension}; the - * protocol compiler generates a static instance of this class for every - * extension in its input. Through the magic of generics, all is made - * type-safe. + *

This class implements type-safe accessors for extensions. They implement all the same + * operations that you can do with normal fields -- e.g. "get", "set", and "add" -- but for + * extensions. The extensions are identified using instances of the class {@link + * GeneratedExtension}; the protocol compiler generates a static instance of this class for every + * extension in its input. Through the magic of generics, all is made type-safe. * *

For example, imagine you have the {@code .proto} file: * @@ -1350,17 +1292,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage */ @SuppressWarnings("unchecked") public abstract static class ExtendableBuilder< - MessageType extends ExtendableMessage, - BuilderType extends ExtendableBuilder> - extends Builder - implements ExtendableMessageOrBuilder { + MessageType extends ExtendableMessage, + BuilderType extends ExtendableBuilder> + extends Builder implements ExtendableMessageOrBuilder { private FieldSet.Builder extensions; protected ExtendableBuilder() {} - protected ExtendableBuilder( - BuilderParent parent) { + protected ExtendableBuilder(BuilderParent parent) { super(parent); } @@ -1381,16 +1321,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - private void verifyExtensionContainingType( - final Extension extension) { - if (extension.getDescriptor().getContainingType() != - getDescriptorForType()) { + private void verifyExtensionContainingType(final Extension extension) { + if (extension.getDescriptor().getContainingType() != getDescriptorForType()) { // This can only happen if someone uses unchecked operations. throw new IllegalArgumentException( - "Extension is for type \"" + - extension.getDescriptor().getContainingType().getFullName() + - "\" which does not match message type \"" + - getDescriptorForType().getFullName() + "\"."); + "Extension is for type \"" + + extension.getDescriptor().getContainingType().getFullName() + + "\" which does not match message type \"" + + getDescriptorForType().getFullName() + + "\"."); } } @@ -1425,12 +1364,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (value == null) { if (descriptor.isRepeated()) { return (Type) Collections.emptyList(); - } else if (descriptor.getJavaType() == - FieldDescriptor.JavaType.MESSAGE) { + } else if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { return (Type) extension.getMessageDefaultInstance(); } else { - return (Type) extension.fromReflectionType( - descriptor.getDefaultValue()); + return (Type) extension.fromReflectionType(descriptor.getDefaultValue()); } } else { return (Type) extension.fromReflectionType(value); @@ -1454,8 +1391,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Set the value of an extension. */ public final BuilderType setExtension( - final ExtensionLite extensionLite, - final Type value) { + final ExtensionLite extensionLite, final Type value) { Extension extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); @@ -1469,30 +1405,27 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Set the value of one element of a repeated extension. */ public final BuilderType setExtension( final ExtensionLite> extensionLite, - final int index, final Type value) { + final int index, + final Type value) { Extension> extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); ensureExtensionsIsMutable(); final FieldDescriptor descriptor = extension.getDescriptor(); - extensions.setRepeatedField( - descriptor, index, - extension.singularToReflectionType(value)); + extensions.setRepeatedField(descriptor, index, extension.singularToReflectionType(value)); onChanged(); return (BuilderType) this; } /** Append a value to a repeated extension. */ public final BuilderType addExtension( - final ExtensionLite> extensionLite, - final Type value) { + final ExtensionLite> extensionLite, final Type value) { Extension> extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); ensureExtensionsIsMutable(); final FieldDescriptor descriptor = extension.getDescriptor(); - extensions.addRepeatedField( - descriptor, extension.singularToReflectionType(value)); + extensions.addRepeatedField(descriptor, extension.singularToReflectionType(value)); onChanged(); return (BuilderType) this; } @@ -1603,8 +1536,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called by the build code path to create a copy of the extensions for - * building the message. + * Called by the build code path to create a copy of the extensions for building the message. */ private FieldSet buildExtensions() { return extensions == null @@ -1694,8 +1626,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeatedField(final FieldDescriptor field, - final int index) { + public Object getRepeatedField(final FieldDescriptor field, final int index) { if (field.isExtension()) { verifyContainingType(field); if (extensions == null) { @@ -1744,8 +1675,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType setField(final FieldDescriptor field, - final Object value) { + public BuilderType setField(final FieldDescriptor field, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1771,8 +1701,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType setRepeatedField(final FieldDescriptor field, - final int index, final Object value) { + public BuilderType setRepeatedField( + final FieldDescriptor field, final int index, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1785,8 +1715,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType addRepeatedField(final FieldDescriptor field, - final Object value) { + public BuilderType addRepeatedField(final FieldDescriptor field, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1817,8 +1746,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private void verifyContainingType(final FieldDescriptor field) { if (field.getContainingType() != getDescriptorForType()) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } } } @@ -1826,8 +1754,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // ----------------------------------------------------------------- /** - * Gets the descriptor for an extension. The implementation depends on whether - * the extension is scoped in the top level of a file or scoped in a Message. + * Gets the descriptor for an extension. The implementation depends on whether the extension is + * scoped in the top level of a file or scoped in a Message. */ static interface ExtensionDescriptorRetriever { FieldDescriptor getDescriptor(); @@ -1844,8 +1772,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return clazz.getMethod(name, params); } catch (NoSuchMethodException e) { throw new RuntimeException( - "Generated message class \"" + clazz.getName() + - "\" missing method \"" + name + "\".", e); + "Generated message class \"" + clazz.getName() + "\" missing method \"" + name + "\".", + e); } } @@ -1857,8 +1785,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return method.invoke(object, params); } catch (IllegalAccessException e) { throw new RuntimeException( - "Couldn't use Java reflection to implement protocol message " + - "reflection.", e); + "Couldn't use Java reflection to implement protocol message " + "reflection.", e); } catch (InvocationTargetException e) { final Throwable cause = e.getCause(); if (cause instanceof RuntimeException) { @@ -1867,7 +1794,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage throw (Error) cause; } else { throw new RuntimeException( - "Unexpected exception thrown by generated accessor method.", cause); + "Unexpected exception thrown by generated accessor method.", cause); } } } @@ -1885,25 +1812,24 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected MapField internalGetMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } /** - * Users should ignore this class. This class provides the implementation - * with access to the fields of a message object using Java reflection. + * Users should ignore this class. This class provides the implementation with access to the + * fields of a message object using Java reflection. */ public static final class FieldAccessorTable { /** - * Construct a FieldAccessorTable for a particular message class. Only - * one FieldAccessorTable should ever be constructed per class. + * Construct a FieldAccessorTable for a particular message class. Only one FieldAccessorTable + * should ever be constructed per class. * - * @param descriptor The type's descriptor. - * @param camelCaseNames The camelcase names of all fields in the message. - * These are used to derive the accessor method names. - * @param messageClass The message type. - * @param builderClass The builder type. + * @param descriptor The type's descriptor. + * @param camelCaseNames The camelcase names of all fields in the message. These are used to + * derive the accessor method names. + * @param messageClass The message type. + * @param builderClass The builder type. */ public FieldAccessorTable( final Descriptor descriptor, @@ -1915,12 +1841,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Construct a FieldAccessorTable for a particular message class without - * initializing FieldAccessors. + * Construct a FieldAccessorTable for a particular message class without initializing + * FieldAccessors. */ - public FieldAccessorTable( - final Descriptor descriptor, - final String[] camelCaseNames) { + public FieldAccessorTable(final Descriptor descriptor, final String[] camelCaseNames) { this.descriptor = descriptor; this.camelCaseNames = camelCaseNames; fields = new FieldAccessor[descriptor.getFields().size()]; @@ -1931,16 +1855,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Ensures the field accessors are initialized. This method is thread-safe. * - * @param messageClass The message type. - * @param builderClass The builder type. + * @param messageClass The message type. + * @param builderClass The builder type. * @return this */ public FieldAccessorTable ensureFieldAccessorsInitialized( - Class messageClass, - Class builderClass) { - if (initialized) { return this; } + Class messageClass, Class builderClass) { + if (initialized) { + return this; + } synchronized (this) { - if (initialized) { return this; } + if (initialized) { + return this; + } int fieldsSize = fields.length; for (int i = 0; i < fieldsSize; i++) { FieldDescriptor field = descriptor.getFields().get(i); @@ -1952,36 +1879,54 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (field.isRepeated()) { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isMapField()) { - fields[i] = new MapFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new MapFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = new RepeatedMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedMessageFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); } } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new RepeatedEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedEnumFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = new RepeatedFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); } } else { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - fields[i] = new SingularMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularMessageFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new SingularEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularEnumFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else if (field.getJavaType() == FieldDescriptor.JavaType.STRING) { - fields[i] = new SingularStringFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularStringFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else { - fields[i] = new SingularFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } } } @@ -2007,13 +1952,11 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Get the FieldAccessor for a particular field. */ private FieldAccessor getField(final FieldDescriptor field) { if (field.getContainingType() != descriptor) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } else if (field.isExtension()) { // If this type had extensions, it would subclass ExtendableMessage, // which overrides the reflection interface to handle extensions. - throw new IllegalArgumentException( - "This type does not have extensions."); + throw new IllegalArgumentException("This type does not have extensions."); } return fields[field.getIndex()]; } @@ -2021,38 +1964,53 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Get the OneofAccessor for a particular oneof. */ private OneofAccessor getOneof(final OneofDescriptor oneof) { if (oneof.getContainingType() != descriptor) { - throw new IllegalArgumentException( - "OneofDescriptor does not match message type."); + throw new IllegalArgumentException("OneofDescriptor does not match message type."); } return oneofs[oneof.getIndex()]; } /** - * Abstract interface that provides access to a single field. This is - * implemented differently depending on the field type and cardinality. + * Abstract interface that provides access to a single field. This is implemented differently + * depending on the field type and cardinality. */ private interface FieldAccessor { Object get(GeneratedMessageV3 message); + Object get(GeneratedMessageV3.Builder builder); + Object getRaw(GeneratedMessageV3 message); + Object getRaw(GeneratedMessageV3.Builder builder); + void set(Builder builder, Object value); + Object getRepeated(GeneratedMessageV3 message, int index); + Object getRepeated(GeneratedMessageV3.Builder builder, int index); + Object getRepeatedRaw(GeneratedMessageV3 message, int index); + Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index); - void setRepeated(Builder builder, - int index, Object value); + + void setRepeated(Builder builder, int index, Object value); + void addRepeated(Builder builder, Object value); + boolean has(GeneratedMessageV3 message); + boolean has(GeneratedMessageV3.Builder builder); + int getRepeatedCount(GeneratedMessageV3 message); + int getRepeatedCount(GeneratedMessageV3.Builder builder); + void clear(Builder builder); + Message.Builder newBuilder(); + Message.Builder getBuilder(GeneratedMessageV3.Builder builder); - Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, - int index); + + Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index); } /** OneofAccessor provides access to a single oneof. */ @@ -2280,48 +2238,59 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object get(final GeneratedMessageV3 message) { return invoker.get(message); } + @Override public Object get(GeneratedMessageV3.Builder builder) { return invoker.get(builder); } + @Override public Object getRaw(final GeneratedMessageV3 message) { return get(message); } + @Override public Object getRaw(GeneratedMessageV3.Builder builder) { return get(builder); } + @Override public void set(final Builder builder, final Object value) { invoker.set(builder, value); } + @Override public Object getRepeated(final GeneratedMessageV3 message, final int index) { throw new UnsupportedOperationException("getRepeatedField() called on a singular field."); } + @Override public Object getRepeatedRaw(final GeneratedMessageV3 message, final int index) { throw new UnsupportedOperationException( "getRepeatedFieldRaw() called on a singular field."); } + @Override public Object getRepeated(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException("getRepeatedField() called on a singular field."); } + @Override public Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( "getRepeatedFieldRaw() called on a singular field."); } + @Override public void setRepeated(final Builder builder, final int index, final Object value) { throw new UnsupportedOperationException("setRepeatedField() called on a singular field."); } + @Override public void addRepeated(final Builder builder, final Object value) { throw new UnsupportedOperationException("addRepeatedField() called on a singular field."); } + @Override public boolean has(final GeneratedMessageV3 message) { if (!hasHasMethod) { @@ -2332,6 +2301,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } return invoker.has(message); } + @Override public boolean has(GeneratedMessageV3.Builder builder) { if (!hasHasMethod) { @@ -2342,29 +2312,35 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } return invoker.has(builder); } + @Override public int getRepeatedCount(final GeneratedMessageV3 message) { throw new UnsupportedOperationException( "getRepeatedFieldSize() called on a singular field."); } + @Override public int getRepeatedCount(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException( "getRepeatedFieldSize() called on a singular field."); } + @Override public void clear(final Builder builder) { invoker.clear(builder); } + @Override public Message.Builder newBuilder() { throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("getFieldBuilder() called on a non-Message type."); } + @Override public Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( @@ -2435,8 +2411,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeated( - final GeneratedMessageV3 message, final int index) { + public Object getRepeated(final GeneratedMessageV3 message, final int index) { return invokeOrDie(getRepeatedMethod, message, index); } @@ -2453,8 +2428,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public void addRepeated( - final GeneratedMessageV3.Builder builder, final Object value) { + public void addRepeated(final GeneratedMessageV3.Builder builder, final Object value) { // TODO(b/230609037): remove the unused variable Object unused = invokeOrDie(addRepeatedMethod, builder, value); } @@ -2480,7 +2454,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected final MethodInvoker invoker; RepeatedFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { ReflectionInvoker reflectionInvoker = @@ -2497,18 +2472,22 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object get(final GeneratedMessageV3 message) { return invoker.get(message); } + @Override public Object get(GeneratedMessageV3.Builder builder) { return invoker.get(builder); } + @Override public Object getRaw(final GeneratedMessageV3 message) { return get(message); } + @Override public Object getRaw(GeneratedMessageV3.Builder builder) { return get(builder); } + @Override public void set(final Builder builder, final Object value) { // Add all the elements individually. This serves two purposes: @@ -2520,59 +2499,73 @@ public abstract class GeneratedMessageV3 extends AbstractMessage addRepeated(builder, element); } } + @Override public Object getRepeated(final GeneratedMessageV3 message, final int index) { return invoker.getRepeated(message, index); } + @Override public Object getRepeated(GeneratedMessageV3.Builder builder, int index) { return invoker.getRepeated(builder, index); } + @Override public Object getRepeatedRaw(GeneratedMessageV3 message, int index) { return getRepeated(message, index); } + @Override public Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index) { return getRepeated(builder, index); } + @Override public void setRepeated(final Builder builder, final int index, final Object value) { invoker.setRepeated(builder, index, value); } + @Override public void addRepeated(final Builder builder, final Object value) { invoker.addRepeated(builder, value); } + @Override public boolean has(final GeneratedMessageV3 message) { throw new UnsupportedOperationException("hasField() called on a repeated field."); } + @Override public boolean has(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("hasField() called on a repeated field."); } + @Override public int getRepeatedCount(final GeneratedMessageV3 message) { return invoker.getRepeatedCount(message); } + @Override public int getRepeatedCount(GeneratedMessageV3.Builder builder) { return invoker.getRepeatedCount(builder); } + @Override public void clear(final Builder builder) { invoker.clear(builder); } + @Override public Message.Builder newBuilder() { throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("getFieldBuilder() called on a non-Message type."); } + @Override public Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( @@ -2582,16 +2575,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private static class MapFieldAccessor implements FieldAccessor { MapFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { field = descriptor; - Method getDefaultInstanceMethod = - getMethodOrDie(messageClass, "getDefaultInstance"); - MapField defaultMapField = getMapField( - (GeneratedMessageV3) invokeOrDie(getDefaultInstanceMethod, null)); - mapEntryMessageDefaultInstance = - defaultMapField.getMapEntryMessageDefaultInstance(); + Method getDefaultInstanceMethod = getMethodOrDie(messageClass, "getDefaultInstance"); + MapField defaultMapField = + getMapField((GeneratedMessageV3) invokeOrDie(getDefaultInstanceMethod, null)); + mapEntryMessageDefaultInstance = defaultMapField.getMapEntryMessageDefaultInstance(); } private final FieldDescriptor field; @@ -2605,10 +2597,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return (MapField) builder.internalGetMapField(field.getNumber()); } - private MapField getMutableMapField( - GeneratedMessageV3.Builder builder) { - return (MapField) builder.internalGetMutableMapField( - field.getNumber()); + private MapField getMutableMapField(GeneratedMessageV3.Builder builder) { + return (MapField) builder.internalGetMutableMapField(field.getNumber()); } private Message coerceType(Message value) { @@ -2695,14 +2685,12 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public boolean has(GeneratedMessageV3 message) { - throw new UnsupportedOperationException( - "hasField() is not supported for repeated fields."); + throw new UnsupportedOperationException("hasField() is not supported for repeated fields."); } @Override public boolean has(Builder builder) { - throw new UnsupportedOperationException( - "hasField() is not supported for repeated fields."); + throw new UnsupportedOperationException("hasField() is not supported for repeated fields."); } @Override @@ -2727,8 +2715,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public com.google.protobuf.Message.Builder getBuilder(Builder builder) { - throw new UnsupportedOperationException( - "Nested builder not supported for map fields."); + throw new UnsupportedOperationException("Nested builder not supported for map fields."); } @Override @@ -2739,10 +2726,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // --------------------------------------------------------------- - private static final class SingularEnumFieldAccessor - extends SingularFieldAccessor { + private static final class SingularEnumFieldAccessor extends SingularFieldAccessor { SingularEnumFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { @@ -2755,12 +2742,9 @@ public abstract class GeneratedMessageV3 extends AbstractMessage supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); if (supportUnknownEnumValue) { - getValueMethod = - getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); - getValueMethodBuilder = - getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); - setValueMethod = - getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class); + getValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); + getValueMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); + setValueMethod = getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class); } } @@ -2804,10 +2788,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - private static final class RepeatedEnumFieldAccessor - extends RepeatedFieldAccessor { + private static final class RepeatedEnumFieldAccessor extends RepeatedFieldAccessor { RepeatedEnumFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { super(descriptor, camelCaseName, messageClass, builderClass); @@ -2829,6 +2813,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage getMethodOrDie(builderClass, "add" + camelCaseName + "Value", int.class); } } + private EnumDescriptor enumDescriptor; private final Method valueOfMethod; @@ -2894,6 +2879,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } super.setRepeated(builder, index, invokeOrDie(valueOfMethod, null, value)); } + @Override public void addRepeated(final Builder builder, final Object value) { if (supportUnknownEnumValue) { @@ -2912,29 +2898,25 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Field accessor for string fields. * - *

This class makes getFooBytes() and setFooBytes() available for - * reflection API so that reflection based serialize/parse functions can - * access the raw bytes of the field to preserve non-UTF8 bytes in the - * string. + *

This class makes getFooBytes() and setFooBytes() available for reflection API so that + * reflection based serialize/parse functions can access the raw bytes of the field to preserve + * non-UTF8 bytes in the string. * - *

This ensures the serialize/parse round-trip safety, which is important - * for servers which forward messages. + *

This ensures the serialize/parse round-trip safety, which is important for servers which + * forward messages. */ - private static final class SingularStringFieldAccessor - extends SingularFieldAccessor { + private static final class SingularStringFieldAccessor extends SingularFieldAccessor { SingularStringFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { - super(descriptor, camelCaseName, messageClass, builderClass, - containingOneofCamelCaseName); - getBytesMethod = getMethodOrDie(messageClass, - "get" + camelCaseName + "Bytes"); - getBytesMethodBuilder = getMethodOrDie(builderClass, - "get" + camelCaseName + "Bytes"); - setBytesMethodBuilder = getMethodOrDie(builderClass, - "set" + camelCaseName + "Bytes", ByteString.class); + super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); + getBytesMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Bytes"); + getBytesMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Bytes"); + setBytesMethodBuilder = + getMethodOrDie(builderClass, "set" + camelCaseName + "Bytes", ByteString.class); } private final Method getBytesMethod; @@ -2964,19 +2946,17 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // --------------------------------------------------------------- - private static final class SingularMessageFieldAccessor - extends SingularFieldAccessor { + private static final class SingularMessageFieldAccessor extends SingularFieldAccessor { SingularMessageFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { - super(descriptor, camelCaseName, messageClass, builderClass, - containingOneofCamelCaseName); + super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); newBuilderMethod = getMethodOrDie(type, "newBuilder"); - getBuilderMethodBuilder = - getMethodOrDie(builderClass, "get" + camelCaseName + "Builder"); + getBuilderMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Builder"); } private final Method newBuilderMethod; @@ -3000,27 +2980,29 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public void set(final Builder builder, final Object value) { super.set(builder, coerceType(value)); } + @Override public Message.Builder newBuilder() { return (Message.Builder) invokeOrDie(newBuilderMethod, null); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { return (Message.Builder) invokeOrDie(getBuilderMethodBuilder, builder); } } - private static final class RepeatedMessageFieldAccessor - extends RepeatedFieldAccessor { + private static final class RepeatedMessageFieldAccessor extends RepeatedFieldAccessor { RepeatedMessageFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { super(descriptor, camelCaseName, messageClass, builderClass); newBuilderMethod = getMethodOrDie(type, "newBuilder"); - getBuilderMethodBuilder = getMethodOrDie(builderClass, - "get" + camelCaseName + "Builder", Integer.TYPE); + getBuilderMethodBuilder = + getMethodOrDie(builderClass, "get" + camelCaseName + "Builder", Integer.TYPE); } private final Method newBuilderMethod; @@ -3044,27 +3026,30 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public void setRepeated(final Builder builder, final int index, final Object value) { super.setRepeated(builder, index, coerceType(value)); } + @Override public void addRepeated(final Builder builder, final Object value) { super.addRepeated(builder, coerceType(value)); } + @Override public Message.Builder newBuilder() { return (Message.Builder) invokeOrDie(newBuilderMethod, null); } + @Override public Message.Builder getRepeatedBuilder( final GeneratedMessageV3.Builder builder, final int index) { - return (Message.Builder) invokeOrDie( - getBuilderMethodBuilder, builder, index); + return (Message.Builder) invokeOrDie(getBuilderMethodBuilder, builder, index); } } } /** - * Replaces this object in the output stream with a serialized form. - * Part of Java's serialization magic. Generated sub-classes must override - * this method by calling {@code return super.writeReplace();} + * Replaces this object in the output stream with a serialized form. Part of Java's serialization + * magic. Generated sub-classes must override this method by calling {@code return + * super.writeReplace();} + * * @return a SerializedForm of this message */ protected Object writeReplace() throws ObjectStreamException { @@ -3116,8 +3101,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static void writeStringNoTag( - CodedOutputStream output, final Object value) throws IOException { + protected static void writeStringNoTag(CodedOutputStream output, final Object value) + throws IOException { if (value instanceof String) { output.writeStringNoTag((String) value); } else { @@ -3129,7 +3114,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedOutputStream out, MapField field, MapEntry defaultEntry, - int fieldNumber) throws IOException { + int fieldNumber) + throws IOException { Map m = field.getMap(); if (!out.isSerializationDeterministic()) { serializeMapTo(out, m, defaultEntry, fieldNumber); @@ -3144,11 +3130,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } Arrays.sort(keys); for (int key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3171,11 +3154,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } Arrays.sort(keys); for (long key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3197,11 +3177,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage keys = m.keySet().toArray(keys); Arrays.sort(keys); for (String key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3228,28 +3205,23 @@ public abstract class GeneratedMessageV3 extends AbstractMessage boolean key) throws IOException { if (m.containsKey(key)) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } /** Serialize the map using the iteration order. */ private static void serializeMapTo( - CodedOutputStream out, - Map m, - MapEntry defaultEntry, - int fieldNumber) + CodedOutputStream out, Map m, MapEntry defaultEntry, int fieldNumber) throws IOException { for (Map.Entry entry : m.entrySet()) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() + out.writeMessage( + fieldNumber, + defaultEntry + .newBuilderForType() .setKey(entry.getKey()) .setValue(entry.getValue()) .build()); } } } - diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index a88baca61c..7180ce3fc0 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -463,15 +463,15 @@ public class DescriptorsTest { /** Tests that parsing an unknown enum throws an exception */ @Test public void testParseUnknownEnum() { - FieldDescriptorProto.Builder field = FieldDescriptorProto.newBuilder() - .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) - .setTypeName("UnknownEnum") - .setType(FieldDescriptorProto.Type.TYPE_ENUM) - .setName("bar") - .setNumber(1); - DescriptorProto.Builder messageType = DescriptorProto.newBuilder() - .setName("Foo") - .addField(field); + FieldDescriptorProto.Builder field = + FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("UnknownEnum") + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setName("bar") + .setNumber(1); + DescriptorProto.Builder messageType = + DescriptorProto.newBuilder().setName("Foo").addField(field); FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() .setName("foo.proto") @@ -486,7 +486,6 @@ public class DescriptorsTest { } } - /** * Tests the translate/crosslink for an example where a message field's name and type name are the * same. diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index f4c4baf4a2..01c2f4b39a 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -1943,7 +1943,7 @@ public class GeneratedMessageTest { @Test public void - extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() { + extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() { TestAllExtensions.Builder builder = TestAllExtensions.newBuilder(); builder.addRepeatedField(REPEATED_NESTED_MESSAGE_EXTENSION, NestedMessage.getDefaultInstance()); // Calling getRepeatedFieldBuilder and ignoring the returned Builder should have no diff --git a/objectivec/README.md b/objectivec/README.md index a0fcd19307..f2c2a846a7 100644 --- a/objectivec/README.md +++ b/objectivec/README.md @@ -200,12 +200,13 @@ supported keys are: entry can be made as "no_package:PATH=prefix", where PATH is the path for the .proto file. - * `use_package_as_prefix` and `proto_package_prefix_exceptions_path`: The - `value` for `use_package_as_prefix` can be `yes` or `no`, and indicates - if a prefix should be derived from the proto package for all the symbols - for files that don't have the `objc_class_prefix` file option (mentioned - above). This helps ensure the symbols are more unique and means there is - less chance of ObjC class name collisions. + * `use_package_as_prefix`, `package_as_prefix_forced_prefix` and + `proto_package_prefix_exceptions_path`: The `value` for + `use_package_as_prefix` can be `yes` or `no`, and indicates if a prefix + should be derived from the proto package for all the symbols for files that + don't have the `objc_class_prefix` file option (mentioned above). This helps + ensure the symbols are more unique and means there is less chance of ObjC + class name collisions. To help in migrating code to using this support, `proto_package_prefix_exceptions_path` can be used to provide the path @@ -213,10 +214,16 @@ supported keys are: if prefixed with `#`). These package won't get the derived prefix, allowing migrations to the behavior one proto package at a time across a code base. + `package_as_prefix_forced_prefix` can be used to provide a value that will + be used before all prefixes derived from the packages to help group all of + these types with a common prefix. Thus it only makes sense to use it when + `use_package_as_prefix` is also enabled. For example, setting this to + "XYZ\_" and generating a file with the package "something" defining + "MyMessage", would have Objective-C class be `XYZ_Something_MyMessage`. + `use_package_as_prefix` currently defaults to `no` (existing behavior), but - in the future (as a breaking change), that is likely to change since it - helps prepare folks before they end up using a lot of protos and getting a - lot of collisions. + that could change in the future as it helps avoid collisions when more + protos get added to the build. Note that this would be a breaking change. * `headers_use_forward_declarations`: The `value` for this can be `yes` or `no`, and indicates if the generated headers use forward declarations for diff --git a/php/src/Google/Protobuf/FieldDescriptor.php b/php/src/Google/Protobuf/FieldDescriptor.php index 6d08cea9da..ac919a24a9 100644 --- a/php/src/Google/Protobuf/FieldDescriptor.php +++ b/php/src/Google/Protobuf/FieldDescriptor.php @@ -39,6 +39,7 @@ class FieldDescriptor { use GetPublicDescriptorTrait; + /** @var \Google\Protobuf\Internal\FieldDescriptor $internal_desc */ private $internal_desc; /** @@ -81,6 +82,32 @@ class FieldDescriptor return $this->internal_desc->getType(); } + /** + * @return OneofDescriptor + */ + public function getContainingOneof() + { + return $this->getPublicDescriptor($this->internal_desc->getContainingOneof()); + } + + /** + * Gets the field's containing oneof, only if non-synthetic. + * + * @return null|OneofDescriptor + */ + public function getRealContainingOneof() + { + return $this->getPublicDescriptor($this->internal_desc->getRealContainingOneof()); + } + + /** + * @return boolean + */ + public function hasOptionalKeyword() + { + return $this->internal_desc->hasOptionalKeyword(); + } + /** * @return Descriptor Returns a descriptor for the field type if the field type is a message, otherwise throws \Exception * @throws \Exception @@ -114,12 +141,4 @@ class FieldDescriptor { return $this->internal_desc->isMap(); } - - /** - * @return boolean - */ - public function hasOptionalKeyword() - { - return $this->internal_desc->hasOptionalKeyword(); - } } diff --git a/php/src/Google/Protobuf/Internal/FieldDescriptor.php b/php/src/Google/Protobuf/Internal/FieldDescriptor.php index ce83f63a2b..3a9a73b729 100644 --- a/php/src/Google/Protobuf/Internal/FieldDescriptor.php +++ b/php/src/Google/Protobuf/Internal/FieldDescriptor.php @@ -46,8 +46,11 @@ class FieldDescriptor private $message_type; private $enum_type; private $packed; - private $is_map; private $oneof_index = -1; + private $proto3_optional; + + /** @var OneofDescriptor $containing_oneof */ + private $containing_oneof; public function __construct() { @@ -169,6 +172,32 @@ class FieldDescriptor return $this->packed; } + public function getProto3Optional() + { + return $this->proto3_optional; + } + + public function setProto3Optional($proto3_optional) + { + $this->proto3_optional = $proto3_optional; + } + + public function getContainingOneof() + { + return $this->containing_oneof; + } + + public function setContainingOneof($containing_oneof) + { + $this->containing_oneof = $containing_oneof; + } + + public function getRealContainingOneof() + { + return !is_null($this->containing_oneof) && !$this->containing_oneof->isSynthetic() + ? $this->containing_oneof : null; + } + public function isPackable() { return $this->isRepeated() && self::isTypePackable($this->type); @@ -214,6 +243,10 @@ class FieldDescriptor $field_type !== GPBType::BYTES); } + /** + * @param FieldDescriptorProto $proto + * @return FieldDescriptor + */ public static function getFieldDescriptor($proto) { $type_name = null; @@ -248,8 +281,6 @@ class FieldDescriptor $field = new FieldDescriptor(); $field->setName($proto->getName()); - $json_name = $proto->hasJsonName() ? $proto->getJsonName() : - lcfirst(implode('', array_map('ucwords', explode('_', $proto->getName())))); if ($proto->hasJsonName()) { $json_name = $proto->getJsonName(); } else { @@ -269,6 +300,7 @@ class FieldDescriptor $field->setLabel($proto->getLabel()); $field->setPacked($packed); $field->setOneofIndex($oneof_index); + $field->setProto3Optional($proto->getProto3Optional()); // At this time, the message/enum type may have not been added to pool. // So we use the type name as place holder and will replace it with the diff --git a/php/src/Google/Protobuf/Internal/OneofDescriptor.php b/php/src/Google/Protobuf/Internal/OneofDescriptor.php index 67b107f6a4..4323685710 100644 --- a/php/src/Google/Protobuf/Internal/OneofDescriptor.php +++ b/php/src/Google/Protobuf/Internal/OneofDescriptor.php @@ -37,6 +37,7 @@ class OneofDescriptor use HasPublicDescriptorTrait; private $name; + /** @var \Google\Protobuf\FieldDescriptor[] $fields */ private $fields; public function __construct() @@ -64,13 +65,21 @@ class OneofDescriptor return $this->fields; } + public function isSynthetic() + { + return !is_null($this->fields) && count($this->fields) === 1 + && $this->fields[0]->getProto3Optional(); + } + public static function buildFromProto($oneof_proto, $desc, $index) { $oneof = new OneofDescriptor(); $oneof->setName($oneof_proto->getName()); foreach ($desc->getField() as $field) { + /** @var FieldDescriptor $field */ if ($field->getOneofIndex() == $index) { $oneof->addField($field); + $field->setContainingOneof($oneof); } } return $oneof; diff --git a/php/src/Google/Protobuf/OneofDescriptor.php b/php/src/Google/Protobuf/OneofDescriptor.php index 92b4e279da..66ffbd5caf 100644 --- a/php/src/Google/Protobuf/OneofDescriptor.php +++ b/php/src/Google/Protobuf/OneofDescriptor.php @@ -38,6 +38,7 @@ class OneofDescriptor { use GetPublicDescriptorTrait; + /** @var \Google\Protobuf\Internal\OneofDescriptor $internal_desc */ private $internal_desc; /** @@ -62,6 +63,12 @@ class OneofDescriptor */ public function getField($index) { + if ( + is_null($this->internal_desc->getFields()) + || !isset($this->internal_desc->getFields()[$index]) + ) { + return null; + } return $this->getPublicDescriptor($this->internal_desc->getFields()[$index]); } @@ -75,6 +82,6 @@ class OneofDescriptor public function isSynthetic() { - return $this->internal_desc->isSynthetic(); + return $this->internal_desc->isSynthetic(); } } diff --git a/src/Makefile.am b/src/Makefile.am index f88b75e6f4..42975d11e2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -429,6 +429,8 @@ libprotoc_la_SOURCES = \ google/protobuf/compiler/java/message_field_lite.h \ google/protobuf/compiler/java/message_lite.cc \ google/protobuf/compiler/java/message_lite.h \ + google/protobuf/compiler/java/message_serialization.cc \ + google/protobuf/compiler/java/message_serialization.h \ google/protobuf/compiler/java/name_resolver.cc \ google/protobuf/compiler/java/name_resolver.h \ google/protobuf/compiler/java/options.h \ diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 620755aa29..c849a6236e 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -369,6 +369,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field_lite.cc @@ -461,6 +462,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/names.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/options.h diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h index d02ad93985..76727688b5 100644 --- a/src/google/protobuf/arena_impl.h +++ b/src/google/protobuf/arena_impl.h @@ -55,6 +55,13 @@ namespace google { namespace protobuf { namespace internal { +// To prevent sharing cache lines between threads +#ifdef __cpp_aligned_new +enum { kCacheAlignment = 64 }; +#else +enum { kCacheAlignment = alignof(max_align_t) }; // do the best we can +#endif + inline constexpr size_t AlignUpTo8(size_t n) { // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.) return (n + 7) & static_cast(-8); @@ -497,10 +504,10 @@ class PROTOBUF_EXPORT ThreadSafeArena { // have fallback function calls in tail position. This substantially improves // code for the happy path. PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) { - SerialArena* a; + SerialArena* arena; if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && - GetSerialArenaFromThreadCache(&a))) { - return a->MaybeAllocateAligned(n, out); + GetSerialArenaFromThreadCache(&arena))) { + return arena->MaybeAllocateAligned(n, out); } return false; } @@ -564,7 +571,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { // fast path optimizes the case where a single thread uses multiple arenas. ThreadCache* tc = &thread_cache(); SerialArena* serial = hint_.load(std::memory_order_acquire); - if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { + if (PROTOBUF_PREDICT_TRUE(serial != nullptr && serial->owner() == tc)) { *arena = serial; return true; } @@ -602,7 +609,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { #ifdef _MSC_VER #pragma warning(disable : 4324) #endif - struct alignas(64) ThreadCache { + struct alignas(kCacheAlignment) ThreadCache { #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // If we are using the ThreadLocalStorage class to store the ThreadCache, // then the ThreadCache's default constructor has to be responsible for @@ -610,7 +617,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { ThreadCache() : next_lifecycle_id(0), last_lifecycle_id_seen(-1), - last_serial_arena(NULL) {} + last_serial_arena(nullptr) {} #endif // Number of per-thread lifecycle IDs to reserve. Must be power of two. @@ -633,7 +640,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { #ifdef _MSC_VER #pragma warning(disable : 4324) #endif - struct alignas(64) CacheAlignedLifecycleIdGenerator { + struct alignas(kCacheAlignment) CacheAlignedLifecycleIdGenerator { std::atomic id; }; static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_; diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index d6bef1b273..65da0d3d4d 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -200,7 +200,7 @@ std::string* ArenaStringPtr::Release() { if (IsDefault()) return nullptr; std::string* released = tagged_ptr_.Get(); - if (!tagged_ptr_.IsAllocated()) { + if (tagged_ptr_.IsArena()) { released = tagged_ptr_.IsMutable() ? new std::string(std::move(*released)) : new std::string(*released); } @@ -229,9 +229,7 @@ void ArenaStringPtr::SetAllocated(std::string* value, Arena* arena) { } void ArenaStringPtr::Destroy() { - if (tagged_ptr_.IsAllocated()) { - delete tagged_ptr_.Get(); - } + delete tagged_ptr_.GetIfAllocated(); } void ArenaStringPtr::ClearToEmpty() { diff --git a/src/google/protobuf/arenastring.h b/src/google/protobuf/arenastring.h index b8d31fec01..6bc8395f23 100644 --- a/src/google/protobuf/arenastring.h +++ b/src/google/protobuf/arenastring.h @@ -96,13 +96,12 @@ class PROTOBUF_EXPORT LazyString { class TaggedStringPtr { public: - // Bit flags qualifying string properties. We can use up to 3 bits as - // ptr_ is guaranteed and enforced to be aligned on 8 byte boundaries. + // Bit flags qualifying string properties. We can use 2 bits as + // ptr_ is guaranteed and enforced to be aligned on 4 byte boundaries. enum Flags { kArenaBit = 0x1, // ptr is arena allocated - kAllocatedBit = 0x2, // ptr is heap allocated - kMutableBit = 0x4, // ptr contents are fully mutable - kMask = 0x7 // Bit mask + kMutableBit = 0x2, // ptr contents are fully mutable + kMask = 0x3 // Bit mask }; // Composed logical types @@ -112,7 +111,7 @@ class TaggedStringPtr { // Allocated strings are mutable and (as the name implies) owned. // A heap allocated string must be deleted. - kAllocated = kAllocatedBit | kMutableBit, + kAllocated = kMutableBit, // Mutable arena strings are strings where the string instance is owned // by the arena, but the string contents itself are owned by the string @@ -166,8 +165,16 @@ class TaggedStringPtr { // Returns true if the current string is an immutable default value. inline bool IsDefault() const { return (as_int() & kMask) == kDefault; } - // Returns true if the current string is a heap allocated mutable value. - inline bool IsAllocated() const { return as_int() & kAllocatedBit; } + // If the current string is a heap-allocated mutable value, returns a pointer + // to it. Returns nullptr otherwise. + inline std::string *GetIfAllocated() const { + auto allocated = as_int() ^ kAllocated; + if (allocated & kMask) return nullptr; + + auto ptr = reinterpret_cast(allocated); + PROTOBUF_ASSUME(ptr != nullptr); + return ptr; + } // Returns true if the current string is an arena allocated value. // This means it's either a mutable or fixed size arena string. @@ -224,8 +231,8 @@ static_assert(std::is_trivial::value, // Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and // the field is always manually initialized via method calls. // -// See TaggedPtr for more information about the types of string values being -// held, and the mutable and ownership invariants for each type. +// See TaggedStringPtr for more information about the types of string values +// being held, and the mutable and ownership invariants for each type. struct PROTOBUF_EXPORT ArenaStringPtr { ArenaStringPtr() = default; constexpr ArenaStringPtr(ExplicitlyConstructedArenaString* default_value, diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 7383e3eb3b..d78d9fed2f 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -61,12 +61,13 @@ PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased } // namespace -PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10; +PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = { + .next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10}; -ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); } +ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; -void ThreadSafeArenaStats::PrepareForSampling() { +void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { num_allocations.store(0, std::memory_order_relaxed); num_resets.store(0, std::memory_order_relaxed); bytes_requested.store(0, std::memory_order_relaxed); @@ -74,6 +75,7 @@ void ThreadSafeArenaStats::PrepareForSampling() { bytes_wasted.store(0, std::memory_order_relaxed); max_bytes_allocated.store(0, std::memory_order_relaxed); thread_ids.store(0, std::memory_order_relaxed); + weight = stride; // The inliner makes hardcoded skip_count difficult (especially when combined // with LTO). We use the ability to exclude stacks by regex when encoding // instead. @@ -105,12 +107,15 @@ void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested, info->thread_ids.fetch_or(tid, std::memory_order_relaxed); } -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { - bool first = *next_sample < 0; - *next_sample = g_exponential_biased_generator.GetStride( +ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) { + bool first = sampling_state.next_sample < 0; + const int64_t next_stride = g_exponential_biased_generator.GetStride( g_arenaz_sample_parameter.load(std::memory_order_relaxed)); // Small values of interval are equivalent to just sampling next time. - ABSL_ASSERT(*next_sample >= 1); + ABSL_ASSERT(next_stride >= 1); + sampling_state.next_sample = next_stride; + const int64_t old_stride = + absl::exchange(sampling_state.sample_stride, next_stride); // g_arenaz_enabled can be dynamically flipped, we need to set a threshold low // enough that we will start sampling in a reasonable time, so we just use the @@ -119,11 +124,11 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { // We will only be negative on our first count, so we should just retry in // that case. if (first) { - if (PROTOBUF_PREDICT_TRUE(--*next_sample > 0)) return nullptr; - return SampleSlow(next_sample); + if (PROTOBUF_PREDICT_TRUE(--sampling_state.next_sample > 0)) return nullptr; + return SampleSlow(sampling_state); } - return GlobalThreadSafeArenazSampler().Register(); + return GlobalThreadSafeArenazSampler().Register(old_stride); } void SetThreadSafeArenazEnabled(bool enabled) { @@ -150,7 +155,8 @@ void SetThreadSafeArenazMaxSamples(int32_t max) { void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) { if (next_sample >= 0) { - global_next_sample = next_sample; + global_sampling_state.next_sample = next_sample; + global_sampling_state.sample_stride = next_sample; } else { ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz next sample: %lld", static_cast(next_sample)); // NOLINT(runtime/int) diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index 76698a26a3..31c178a009 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -58,8 +58,11 @@ struct ThreadSafeArenaStats ~ThreadSafeArenaStats(); // Puts the object into a clean state, fills in the logically `const` members, - // blocking for any readers that are currently sampling the object. - void PrepareForSampling() ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); + // blocking for any readers that are currently sampling the object. The + // 'stride' parameter is the number of ThreadSafeArenas that were instantiated + // between this sample and the previous one. + void PrepareForSampling(int64_t stride) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); // These fields are mutated by the various Record* APIs and need to be // thread-safe. @@ -91,7 +94,18 @@ struct ThreadSafeArenaStats } }; -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample); +struct SamplingState { + // Number of ThreadSafeArenas that should be instantiated before the next + // ThreadSafeArena is sampled. This variable is decremented with each + // instantiation. + int64_t next_sample; + // When we make a sampling decision, we record that distance between from the + // previous sample so we can weight each sample. 'distance' here is the + // number of instantiations of ThreadSafeArena. + int64_t sample_stride; +}; + +ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state); void UnsampleSlow(ThreadSafeArenaStats* info); class ThreadSafeArenaStatsHandle { @@ -138,24 +152,27 @@ class ThreadSafeArenaStatsHandle { using ThreadSafeArenazSampler = ::absl::profiling_internal::SampleRecorder; -extern PROTOBUF_THREAD_LOCAL int64_t global_next_sample; +extern PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state; // Returns an RAII sampling handle that manages registration and unregistation // with the global sampler. inline ThreadSafeArenaStatsHandle Sample() { - if (PROTOBUF_PREDICT_TRUE(--global_next_sample > 0)) { + if (PROTOBUF_PREDICT_TRUE(--global_sampling_state.next_sample > 0)) { return ThreadSafeArenaStatsHandle(nullptr); } - return ThreadSafeArenaStatsHandle(SampleSlow(&global_next_sample)); + return ThreadSafeArenaStatsHandle(SampleSlow(global_sampling_state)); } #else + +using SamplingState = int64_t; + struct ThreadSafeArenaStats { static void RecordAllocateStats(ThreadSafeArenaStats*, size_t /*requested*/, size_t /*allocated*/, size_t /*wasted*/) {} }; -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample); +ThreadSafeArenaStats* SampleSlow(SamplingState& next_sample); void UnsampleSlow(ThreadSafeArenaStats* info); class ThreadSafeArenaStatsHandle { diff --git a/src/google/protobuf/arenaz_sampler_test.cc b/src/google/protobuf/arenaz_sampler_test.cc index 1bfec542d6..2588c04a75 100644 --- a/src/google/protobuf/arenaz_sampler_test.cc +++ b/src/google/protobuf/arenaz_sampler_test.cc @@ -64,8 +64,9 @@ std::vector GetBytesAllocated(ThreadSafeArenazSampler* s) { return res; } -ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size) { - auto* info = s->Register(); +ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size, + int64_t stride) { + auto* info = s->Register(stride); assert(info != nullptr); info->bytes_allocated.store(size); return info; @@ -79,8 +80,9 @@ namespace { TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 107; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); EXPECT_EQ(info.num_allocations.load(), 0); EXPECT_EQ(info.num_resets.load(), 0); @@ -88,6 +90,7 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { EXPECT_EQ(info.bytes_allocated.load(), 0); EXPECT_EQ(info.bytes_wasted.load(), 0); EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.weight, kTestStride); info.num_allocations.store(1, std::memory_order_relaxed); info.num_resets.store(1, std::memory_order_relaxed); @@ -96,19 +99,21 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { info.bytes_wasted.store(1, std::memory_order_relaxed); info.max_bytes_allocated.store(1, std::memory_order_relaxed); - info.PrepareForSampling(); + info.PrepareForSampling(2 * kTestStride); EXPECT_EQ(info.num_allocations.load(), 0); EXPECT_EQ(info.num_resets.load(), 0); EXPECT_EQ(info.bytes_requested.load(), 0); EXPECT_EQ(info.bytes_allocated.load(), 0); EXPECT_EQ(info.bytes_wasted.load(), 0); EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.weight, 2 * kTestStride); } TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 458; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); EXPECT_EQ(info.num_allocations.load(), 1); EXPECT_EQ(info.num_resets.load(), 0); @@ -128,8 +133,9 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { TEST(ThreadSafeArenaStatsTest, RecordResetSlow) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 584; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); EXPECT_EQ(info.num_resets.load(), 0); EXPECT_EQ(info.bytes_allocated.load(), 0); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); @@ -143,11 +149,12 @@ TEST(ThreadSafeArenaStatsTest, RecordResetSlow) { TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) { SetThreadSafeArenazEnabled(true); SetThreadSafeArenazSampleParameter(100); + constexpr int64_t kTestStride = 0; for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - ThreadSafeArenaStats* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState sampling_state = {kTestStride, kTestStride}; + ThreadSafeArenaStats* sample = SampleSlow(sampling_state); + EXPECT_GT(sampling_state.next_sample, 0); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } @@ -156,11 +163,12 @@ TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) { TEST(ThreadSafeArenazSamplerTest, LargeSampleParameter) { SetThreadSafeArenazEnabled(true); SetThreadSafeArenazSampleParameter(std::numeric_limits::max()); + constexpr int64_t kTestStride = 0; for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - ThreadSafeArenaStats* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState sampling_state = {kTestStride, kTestStride}; + ThreadSafeArenaStats* sample = SampleSlow(sampling_state); + EXPECT_GT(sampling_state.next_sample, 0); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } @@ -187,7 +195,8 @@ TEST(ThreadSafeArenazSamplerTest, Sample) { TEST(ThreadSafeArenazSamplerTest, Handle) { auto& sampler = GlobalThreadSafeArenazSampler(); - ThreadSafeArenaStatsHandle h(sampler.Register()); + constexpr int64_t kTestStride = 17; + ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride)); auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h); info->bytes_allocated.store(0x12345678, std::memory_order_relaxed); @@ -195,6 +204,7 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { sampler.Iterate([&](const ThreadSafeArenaStats& h) { if (&h == info) { EXPECT_EQ(h.bytes_allocated.load(), 0x12345678); + EXPECT_EQ(h.weight, kTestStride); found = true; } }); @@ -216,10 +226,11 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { TEST(ThreadSafeArenazSamplerTest, Registration) { ThreadSafeArenazSampler sampler; - auto* info1 = Register(&sampler, 1); + constexpr int64_t kTestStride = 100; + auto* info1 = Register(&sampler, 1, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1)); - auto* info2 = Register(&sampler, 2); + auto* info2 = Register(&sampler, 2, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2)); info1->bytes_allocated.store(3); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2)); @@ -231,16 +242,17 @@ TEST(ThreadSafeArenazSamplerTest, Registration) { TEST(ThreadSafeArenazSamplerTest, Unregistration) { ThreadSafeArenazSampler sampler; std::vector infos; + constexpr int64_t kTestStride = 200; for (size_t i = 0; i < 3; ++i) { - infos.push_back(Register(&sampler, i)); + infos.push_back(Register(&sampler, i, kTestStride)); } EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2)); sampler.Unregister(infos[1]); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2)); - infos.push_back(Register(&sampler, 3)); - infos.push_back(Register(&sampler, 4)); + infos.push_back(Register(&sampler, 3, kTestStride)); + infos.push_back(Register(&sampler, 4, kTestStride)); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4)); sampler.Unregister(infos[3]); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4)); @@ -257,18 +269,19 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { ThreadPool pool(10); for (int i = 0; i < 10; ++i) { - pool.Schedule([&sampler, &stop]() { + const int64_t sampling_stride = 11 + i % 3; + pool.Schedule([&sampler, &stop, sampling_stride]() { std::random_device rd; std::mt19937 gen(rd()); std::vector infoz; while (!stop.HasBeenNotified()) { if (infoz.empty()) { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride)); } switch (std::uniform_int_distribution<>(0, 1)(gen)) { case 0: { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride)); break; } case 1: { @@ -277,6 +290,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { ThreadSafeArenaStats* info = infoz[p]; infoz[p] = infoz.back(); infoz.pop_back(); + EXPECT_EQ(info->weight, sampling_stride); sampler.Unregister(info); break; } @@ -292,9 +306,10 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { TEST(ThreadSafeArenazSamplerTest, Callback) { ThreadSafeArenazSampler sampler; + constexpr int64_t kTestStride = 203; - auto* info1 = Register(&sampler, 1); - auto* info2 = Register(&sampler, 2); + auto* info1 = Register(&sampler, 1, kTestStride); + auto* info2 = Register(&sampler, 2, kTestStride); static const ThreadSafeArenaStats* expected; diff --git a/src/google/protobuf/compiler/cpp/enum_field.cc b/src/google/protobuf/compiler/cpp/enum_field.cc index 8ffb699e87..3539a0df41 100644 --- a/src/google/protobuf/compiler/cpp/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/enum_field.cc @@ -252,7 +252,9 @@ void RepeatedEnumFieldGenerator::GeneratePrivateMembers( format("::$proto_ns$::RepeatedField $name$_;\n"); if (descriptor_->is_packed() && HasGeneratedMethods(descriptor_->file(), options_)) { - format("mutable std::atomic $cached_byte_size_name$;\n"); + format( + "mutable ::$proto_ns$::internal::CachedSize " + "$cached_byte_size_name$;\n"); } } @@ -364,7 +366,7 @@ void RepeatedEnumFieldGenerator::GenerateSerializeWithCachedSizesToArray( format( "{\n" " int byte_size = " - "$cached_byte_size_field$.load(std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Get();\n" " if (byte_size > 0) {\n" " target = stream->WriteEnumPacked(\n" " $number$, $field$, byte_size, target);\n" @@ -402,8 +404,7 @@ void RepeatedEnumFieldGenerator::GenerateByteSize(io::Printer* printer) const { "::_pbi::WireFormatLite::Int32Size(static_cast<$int32$>(data_size));\n" "}\n" "int cached_size = ::_pbi::ToCachedSize(data_size);\n" - "$cached_byte_size_field$.store(cached_size,\n" - " std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Set(cached_size);\n" "total_size += data_size;\n"); } else { format("total_size += ($tag_size$UL * count) + data_size;\n"); diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index f4eb744e2f..af463d58ac 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -2101,6 +2101,11 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { "static constexpr int32_t kHasBitsOffset =\n" " 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n"); } + if (descriptor_->real_oneof_decl_count() > 0) { + format( + "static constexpr int32_t kOneofCaseOffset =\n" + " PROTOBUF_FIELD_OFFSET($classtype$, $oneof_case$);\n"); + } for (auto field : FieldRange(descriptor_)) { field_generators_.get(field).GenerateInternalAccessorDeclarations(printer); if (IsFieldStripped(field, options_)) { diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 3254b37581..db4d209777 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -292,23 +292,11 @@ TailCallTableInfo::TailCallTableInfo( const std::vector& has_bit_indices, const std::vector& inlined_string_indices, MessageSCCAnalyzer* scc_analyzer) { - int oneof_count = descriptor->real_oneof_decl_count(); - // If this message has any oneof fields, store the case offset in the first - // auxiliary entry. - if (oneof_count > 0) { - GOOGLE_LOG_IF(DFATAL, ordered_fields.empty()) - << "Invalid message: " << descriptor->full_name() << " has " - << oneof_count << " oneof declarations, but no fields"; - aux_entries.push_back(StrCat("_fl::Offset{offsetof(", - ClassName(descriptor), - ", _impl_._oneof_case_)}")); - } - // If this message has any inlined string fields, store the donation state // offset in the second auxiliary entry. if (!inlined_string_indices.empty()) { - aux_entries.resize(2); // pad if necessary - aux_entries[1] = + aux_entries.resize(1); // pad if necessary + aux_entries[0] = StrCat("_fl::Offset{offsetof(", ClassName(descriptor), ", _impl_._inlined_string_donated_)}"); } @@ -1102,7 +1090,7 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) { FieldMemberName(field, /*cold=*/false)); } if (oneof) { - format("$1$, ", oneof->index()); + format("_Internal::kOneofCaseOffset + $1$, ", 4 * oneof->index()); } else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) { if (entry.hasbit_idx >= 0) { format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx); diff --git a/src/google/protobuf/compiler/cpp/primitive_field.cc b/src/google/protobuf/compiler/cpp/primitive_field.cc index 6c92ede251..342a3b0731 100644 --- a/src/google/protobuf/compiler/cpp/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/primitive_field.cc @@ -328,7 +328,9 @@ void RepeatedPrimitiveFieldGenerator::GeneratePrivateMembers( format("::$proto_ns$::RepeatedField< $type$ > $name$_;\n"); if (descriptor_->is_packed() && FixedSize(descriptor_->type()) == -1 && HasGeneratedMethods(descriptor_->file(), options_)) { - format("mutable std::atomic $cached_byte_size_name$;\n"); + format( + "mutable ::$proto_ns$::internal::CachedSize " + "$cached_byte_size_name$;\n"); } } @@ -433,7 +435,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray( format( "{\n" " int byte_size = " - "$cached_byte_size_field$.load(std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Get();\n" " if (byte_size > 0) {\n" " target = stream->Write$declared_type$Packed(\n" " $number$, _internal_$name$(), byte_size, target);\n" @@ -484,8 +486,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateByteSize( if (FixedSize(descriptor_->type()) == -1) { format( "int cached_size = ::_pbi::ToCachedSize(data_size);\n" - "$cached_byte_size_field$.store(cached_size,\n" - " std::memory_order_relaxed);\n"); + "$cached_byte_size_field$.Set(cached_size);\n"); } format("total_size += data_size;\n"); } else { diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index 95917a7f6d..5adacef3ad 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -53,6 +53,7 @@ #include #include #include +#include #include #include @@ -587,13 +588,6 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( std::unique_ptr sorted_fields( SortFieldsByNumber(descriptor_)); - std::vector sorted_extensions; - sorted_extensions.reserve(descriptor_->extension_range_count()); - for (int i = 0; i < descriptor_->extension_range_count(); ++i) { - sorted_extensions.push_back(descriptor_->extension_range(i)); - } - std::sort(sorted_extensions.begin(), sorted_extensions.end(), - ExtensionRangeOrdering()); printer->Print( "@java.lang.Override\n" "public void writeTo(com.google.protobuf.CodedOutputStream output)\n" @@ -628,19 +622,8 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( } } - // Merge the fields and the extension ranges, both sorted by field number. - for (int i = 0, j = 0; - i < descriptor_->field_count() || j < sorted_extensions.size();) { - if (i == descriptor_->field_count()) { - GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]); - } else if (j == sorted_extensions.size()) { - GenerateSerializeOneField(printer, sorted_fields[i++]); - } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) { - GenerateSerializeOneField(printer, sorted_fields[i++]); - } else { - GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]); - } - } + GenerateSerializeFieldsAndExtensions(printer, field_generators_, descriptor_, + sorted_fields.get()); if (descriptor_->options().message_set_wire_format()) { printer->Print("unknownFields.writeAsMessageSetTo(output);\n"); @@ -770,17 +753,6 @@ void ImmutableMessageGenerator::GenerateParseFromMethods(io::Printer* printer) { GeneratedCodeVersionSuffix()); } -void ImmutableMessageGenerator::GenerateSerializeOneField( - io::Printer* printer, const FieldDescriptor* field) { - field_generators_.get(field).GenerateSerializationCode(printer); -} - -void ImmutableMessageGenerator::GenerateSerializeOneExtensionRange( - io::Printer* printer, const Descriptor::ExtensionRange* range) { - printer->Print("extensionWriter.writeUntil($end$, output);\n", "end", - StrCat(range->end)); -} - // =================================================================== void ImmutableMessageGenerator::GenerateBuilder(io::Printer* printer) { @@ -1017,8 +989,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( " return true;\n" "}\n" "if (!(obj instanceof $classname$)) {\n" - // don't simply return false because mutable and immutable types - // can be equal + // don't simply return false because mutable and immutable types + // can be equal " return super.equals(obj);\n" "}\n" "$classname$ other = ($classname$) obj;\n" diff --git a/src/google/protobuf/compiler/java/message.h b/src/google/protobuf/compiler/java/message.h index 2dbd0dd9bc..45da8ff646 100644 --- a/src/google/protobuf/compiler/java/message.h +++ b/src/google/protobuf/compiler/java/message.h @@ -123,10 +123,6 @@ class ImmutableMessageGenerator : public MessageGenerator { void GenerateMessageSerializationMethods(io::Printer* printer); void GenerateParseFromMethods(io::Printer* printer); - void GenerateSerializeOneField(io::Printer* printer, - const FieldDescriptor* field); - void GenerateSerializeOneExtensionRange( - io::Printer* printer, const Descriptor::ExtensionRange* range); void GenerateBuilder(io::Printer* printer); void GenerateIsInitialized(io::Printer* printer); diff --git a/csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs b/src/google/protobuf/compiler/java/message_serialization.cc similarity index 72% rename from csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs rename to src/google/protobuf/compiler/java/message_serialization.cc index 7b946cb631..5db627d322 100644 --- a/csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs +++ b/src/google/protobuf/compiler/java/message_serialization.cc @@ -1,6 +1,5 @@ -#region Copyright notice and license // Protocol Buffers - Google's data interchange format -// Copyright 2017 Google Inc. All rights reserved. +// Copyright 2008 Google Inc. All rights reserved. // https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without @@ -28,20 +27,24 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion -#if NET35 -using System; -using System.Reflection; +#include -namespace Google.Protobuf.Compatibility -{ - // .NET Core (at least netstandard1.0) doesn't have Delegate.CreateDelegate, and .NET 3.5 doesn't have - // MethodInfo.CreateDelegate. Proxy from one to the other on .NET 3.5... - internal static class MethodInfoExtensions - { - internal static Delegate CreateDelegate(this MethodInfo method, Type type) => - Delegate.CreateDelegate(type, method); - } +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +void GenerateSerializeExtensionRange(io::Printer* printer, + const Descriptor::ExtensionRange* range) { + printer->Print("extensionWriter.writeUntil($end$, output);\n", "end", + StrCat(range->end)); } -#endif + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/java/message_serialization.h b/src/google/protobuf/compiler/java/message_serialization.h new file mode 100644 index 0000000000..6145392f81 --- /dev/null +++ b/src/google/protobuf/compiler/java/message_serialization.h @@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ + +#include +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +// Generates code to serialize a single extension range. +void GenerateSerializeExtensionRange(io::Printer* printer, + const Descriptor::ExtensionRange* range); + +// Generates code to serialize all fields and extension ranges for the specified +// message descriptor, sorting serialization calls in increasing order by field +// number. +// +// Templatized to support different field generator implementations. +template +void GenerateSerializeFieldsAndExtensions( + io::Printer* printer, + const FieldGeneratorMap& field_generators, + const Descriptor* descriptor, const FieldDescriptor** sorted_fields) { + std::vector sorted_extensions; + sorted_extensions.reserve(descriptor->extension_range_count()); + for (int i = 0; i < descriptor->extension_range_count(); ++i) { + sorted_extensions.push_back(descriptor->extension_range(i)); + } + std::sort(sorted_extensions.begin(), sorted_extensions.end(), + ExtensionRangeOrdering()); + + // Merge the fields and the extension ranges, both sorted by field number. + for (int i = 0, j = 0; + i < descriptor->field_count() || j < sorted_extensions.size();) { + if (i == descriptor->field_count()) { + GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); + } else if (j == sorted_extensions.size()) { + field_generators.get(sorted_fields[i++]) + .GenerateSerializationCode(printer); + } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) { + field_generators.get(sorted_fields[i++]) + .GenerateSerializationCode(printer); + } else { + GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); + } + } +} + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ diff --git a/src/google/protobuf/compiler/objectivec/objectivec_file.h b/src/google/protobuf/compiler/objectivec/objectivec_file.h index 5db8f67476..f62a4bc979 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_file.h +++ b/src/google/protobuf/compiler/objectivec/objectivec_file.h @@ -31,9 +31,9 @@ #ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_FILE_H__ #define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_FILE_H__ -#include -#include #include +#include +#include #include #include #include diff --git a/src/google/protobuf/compiler/objectivec/objectivec_generator.cc b/src/google/protobuf/compiler/objectivec/objectivec_generator.cc index 9dccf149a6..d8e4cb5277 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_generator.cc +++ b/src/google/protobuf/compiler/objectivec/objectivec_generator.cc @@ -230,6 +230,10 @@ bool ObjectiveCGenerator::GenerateAll( // - A comment can go on a line after a expected package/prefix pair. // (i.e. - "some.proto.package # comment") SetProtoPackagePrefixExceptionList(options[i].second); + } else if (options[i].first == "package_as_prefix_forced_prefix") { + // String to use as the prefix when deriving a prefix from the package + // name. So this only applies when use_package_as_prefix is also used. + SetForcedPackagePrefix(options[i].second); } else if (options[i].first == "headers_use_forward_declarations") { if (!StringToBool(options[i].second, &generation_options.headers_use_forward_declarations)) { diff --git a/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc b/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc index a4ce84c558..88208b5914 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc +++ b/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc @@ -134,6 +134,7 @@ class PrefixModeStorage { // When using a proto package as the prefix, this should be added as the // prefix in front of it. const std::string& forced_package_prefix() const { return forced_prefix_; } + void set_forced_package_prefix(const std::string& prefix) { forced_prefix_ = prefix; } private: bool use_package_name_; @@ -155,8 +156,6 @@ PrefixModeStorage::PrefixModeStorage() { exception_path_ = exception_path; } - // This one is a not expected to be common, so it doesn't get a generation - // option, just the env var. const char* prefix = getenv("GPB_OBJC_USE_PACKAGE_AS_PREFIX_PREFIX"); if (prefix) { forced_prefix_ = prefix; @@ -254,6 +253,14 @@ void SetProtoPackagePrefixExceptionList(const std::string& file_path) { g_prefix_mode.set_exception_path(file_path); } +std::string GetForcedPackagePrefix() { + return g_prefix_mode.forced_package_prefix(); +} + +void SetForcedPackagePrefix(const std::string& prefix) { + g_prefix_mode.set_forced_package_prefix(prefix); +} + Options::Options() { // While there are generator options, also support env variables to help with // build systems where it isn't as easy to hook in for add the generation diff --git a/src/google/protobuf/compiler/objectivec/objectivec_helpers.h b/src/google/protobuf/compiler/objectivec/objectivec_helpers.h index d21fed215a..a1bcc4ba4a 100644 --- a/src/google/protobuf/compiler/objectivec/objectivec_helpers.h +++ b/src/google/protobuf/compiler/objectivec/objectivec_helpers.h @@ -63,6 +63,10 @@ void PROTOC_EXPORT SetUseProtoPackageAsDefaultPrefix(bool on_or_off); std::string PROTOC_EXPORT GetProtoPackagePrefixExceptionList(); void PROTOC_EXPORT SetProtoPackagePrefixExceptionList( const std::string& file_path); +// Get/Set a prefix to add before the prefix generated from the package name. +// This is only used when UseProtoPackageAsDefaultPrefix() is True. +std::string PROTOC_EXPORT GetForcedPackagePrefix(); +void PROTOC_EXPORT SetForcedPackagePrefix(const std::string& prefix); // Generator Prefix Validation Options (see objectivec_generator.cc for a // description of each): diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 257798e4ae..c77c83c1e9 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -10437,7 +10437,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize( // repeated int32 path = 1 [packed = true]; { - int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._path_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 1, _internal_path(), byte_size, target); @@ -10446,7 +10446,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize( // repeated int32 span = 2 [packed = true]; { - int byte_size = _impl_._span_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._span_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 2, _internal_span(), byte_size, target); @@ -10509,8 +10509,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._path_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._path_cached_byte_size_.Set(cached_size); total_size += data_size; } @@ -10523,8 +10522,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._span_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._span_cached_byte_size_.Set(cached_size); total_size += data_size; } @@ -10996,7 +10994,7 @@ uint8_t* GeneratedCodeInfo_Annotation::_InternalSerialize( // repeated int32 path = 1 [packed = true]; { - int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._path_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 1, _internal_path(), byte_size, target); @@ -11051,8 +11049,7 @@ size_t GeneratedCodeInfo_Annotation::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._path_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._path_cached_byte_size_.Set(cached_size); total_size += data_size; } diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index b8710b274c..84e40775ea 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -7962,9 +7962,9 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_; - mutable std::atomic _path_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > span_; - mutable std::atomic _span_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _span_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField leading_detached_comments_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr leading_comments_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr trailing_comments_; @@ -8350,7 +8350,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_; - mutable std::atomic _path_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr source_file_; int32_t begin_; int32_t end_; diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 1f13e83ccb..a3dd0e2491 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -459,30 +459,36 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } - auto saved_tag = UnalignedLoad(ptr); - ptr += sizeof(TagType); - SyncHasbits(msg, hasbits, table); - auto* aux = table->field_aux(data.aux_idx()); - if (aux_is_table) { - auto* inner_table = aux->table; - auto& field = RefAt(msg, data.offset()); + const auto expected_tag = UnalignedLoad(ptr); + const auto aux = *table->field_aux(data.aux_idx()); + auto& field = RefAt(msg, data.offset()); + do { + ptr += sizeof(TagType); MessageLite* submsg = field.Add>( - inner_table->default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag), - inner_table); + aux_is_table ? aux.table->default_instance : aux.message_default()); + if (aux_is_table) { + if (group_coding) { + ptr = ctx->ParseGroup(submsg, ptr, + FastDecodeTag(expected_tag), aux.table); + } else { + ptr = ctx->ParseMessage(submsg, ptr, aux.table); + } + } else { + if (group_coding) { + ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag)); + } else { + ptr = ctx->ParseMessage(submsg, ptr); + } } - return ctx->ParseMessage(submsg, ptr, inner_table); - } else { - const MessageLite* default_instance = aux->message_default(); - auto& field = RefAt(msg, data.offset()); - MessageLite* submsg = - field.Add>(default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_PASS); } - return ctx->ParseMessage(submsg, ptr); - } + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + } + } while (UnalignedLoad(ptr) == expected_tag); + + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { @@ -1356,11 +1362,8 @@ bool TcParser::ChangeOneof(const TcParseTableBase* table, const TcParseTableBase::FieldEntry& entry, uint32_t field_num, ParseContext* ctx, MessageLite* msg) { - // The _oneof_case_ array offset is stored in the first aux entry. - uint32_t oneof_case_offset = table->field_aux(0u)->offset; - // The _oneof_case_ array index is stored in the has-bit index. - uint32_t* oneof_case = - &TcParser::RefAt(msg, oneof_case_offset) + entry.has_idx; + // The _oneof_case_ value offset is stored in the has-bit index. + uint32_t* oneof_case = &TcParser::RefAt(msg, entry.has_idx); uint32_t current_case = *oneof_case; *oneof_case = field_num; diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index 71d15cdebc..149986d92a 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -185,13 +185,58 @@ T* GetOwnedMessage(Arena* message_arena, T* submessage, // Hide atomic from the public header and allow easy change to regular int // on platforms where the atomic might have a perf impact. +// +// CachedSize is like std::atomic but with some important changes: +// +// 1) CachedSize uses Get / Set rather than load / store. +// 2) CachedSize always uses relaxed ordering. +// 3) CachedSize is assignable and copy-constructible. +// 4) CachedSize has a constexpr default constructor, and a constexpr +// constructor that takes an int argument. +// 5) If the compiler supports the __atomic_load_n / __atomic_store_n builtins, +// then CachedSize is trivially copyable. +// +// Developed at https://godbolt.org/z/vYcx7zYs1 ; supports gcc, clang, MSVC. class PROTOBUF_EXPORT CachedSize { + private: + using Scalar = int; + public: - int Get() const { return size_.load(std::memory_order_relaxed); } - void Set(int size) { size_.store(size, std::memory_order_relaxed); } + constexpr CachedSize() noexcept : atom_(Scalar{}) {} + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr CachedSize(Scalar desired) noexcept : atom_(desired) {} +#if PROTOBUF_BUILTIN_ATOMIC + constexpr CachedSize(const CachedSize& other) = default; + + Scalar Get() const noexcept { + return __atomic_load_n(&atom_, __ATOMIC_RELAXED); + } + + void Set(Scalar desired) noexcept { + __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); + } +#else + CachedSize(const CachedSize& other) noexcept : atom_(other.Get()) {} + CachedSize& operator=(const CachedSize& other) noexcept { + Set(other.Get()); + return *this; + } + + Scalar Get() const noexcept { // + return atom_.load(std::memory_order_relaxed); + } + + void Set(Scalar desired) noexcept { + atom_.store(desired, std::memory_order_relaxed); + } +#endif private: - std::atomic size_{0}; +#if PROTOBUF_BUILTIN_ATOMIC + Scalar atom_; +#else + std::atomic atom_; +#endif }; PROTOBUF_EXPORT void DestroyMessage(const void* message); diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 0b463b8c6a..1da0410fb9 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -139,6 +139,11 @@ #define PROTOBUF_BUILTIN_BSWAP64(x) __builtin_bswap64(x) #endif +// Portable check for gcc-style atomic built-ins +#if __has_builtin(__atomic_load_n) +#define PROTOBUF_BUILTIN_ATOMIC 1 +#endif + // Portable check for GCC minimum version: // https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html #if defined(__GNUC__) && defined(__GNUC_MINOR__) \ diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 44663219f0..184980a7e9 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -38,6 +38,7 @@ #undef PROTOBUF_BUILTIN_BSWAP16 #undef PROTOBUF_BUILTIN_BSWAP32 #undef PROTOBUF_BUILTIN_BSWAP64 +#undef PROTOBUF_BUILTIN_ATOMIC #undef PROTOBUF_GNUC_MIN #undef PROTOBUF_MSC_VER_MIN #undef PROTOBUF_CPLUSPLUS_MIN diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 43c1eae0a3..e746616fb7 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -419,6 +419,8 @@ void Struct::InternalSwap(Struct* other) { class Value::_Internal { public: + static constexpr int32_t kOneofCaseOffset = + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Value, _impl_._oneof_case_); static const ::PROTOBUF_NAMESPACE_ID::Struct& struct_value(const Value* msg); static const ::PROTOBUF_NAMESPACE_ID::ListValue& list_value(const Value* msg); }; diff --git a/src/google/protobuf/stubs/status.cc b/src/google/protobuf/stubs/status.cc index f5c0fa48f1..fc6c7bab0d 100644 --- a/src/google/protobuf/stubs/status.cc +++ b/src/google/protobuf/stubs/status.cc @@ -28,9 +28,10 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include +#include #include -#include #include #include @@ -256,6 +257,12 @@ Status UnknownError(StringPiece message) { return Status(StatusCode::kUnknown, message); } +Status ErrnoToStatus(int error_number, StringPiece message) { + // We will take an Abseil dependency soon, so no reason to do anything + // elaborate here. + return InternalError(StrCat(message, ": ", strerror(error_number))); +} + } // namespace status_internal } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/stubs/status.h b/src/google/protobuf/stubs/status.h index c858cf6239..919e54ae2e 100644 --- a/src/google/protobuf/stubs/status.h +++ b/src/google/protobuf/stubs/status.h @@ -31,10 +31,12 @@ #ifndef GOOGLE_PROTOBUF_STUBS_STATUS_H_ #define GOOGLE_PROTOBUF_STUBS_STATUS_H_ -#include - #include +#include + +#include +// Must be included last. #include namespace google { @@ -147,6 +149,8 @@ PROTOBUF_EXPORT Status UnavailableError(StringPiece message); PROTOBUF_EXPORT Status UnimplementedError(StringPiece message); PROTOBUF_EXPORT Status UnknownError(StringPiece message); +PROTOBUF_EXPORT Status ErrnoToStatus(int error_number, StringPiece message); + } // namespace status_internal using ::google::protobuf::util::status_internal::Status; @@ -187,6 +191,8 @@ using ::google::protobuf::util::status_internal::UnavailableError; using ::google::protobuf::util::status_internal::UnimplementedError; using ::google::protobuf::util::status_internal::UnknownError; +using ::google::protobuf::util::status_internal::ErrnoToStatus; + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 594c8eac6a..b0b4c69fa3 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -129,11 +129,10 @@ void StripWhitespace(std::string *str) { // it only replaces the first instance of "old." // ---------------------------------------------------------------------- -void StringReplace(const std::string &s, const std::string &oldsub, - const std::string &newsub, bool replace_all, - std::string *res) { +void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, + bool replace_all, std::string *res) { if (oldsub.empty()) { - res->append(s); // if empty, append the given string. + StrAppend(res, s); // if empty, append the given string. return; } @@ -144,11 +143,11 @@ void StringReplace(const std::string &s, const std::string &oldsub, if (pos == std::string::npos) { break; } - res->append(s, start_pos, pos - start_pos); - res->append(newsub); + StrAppend(res, s.substr(start_pos, pos - start_pos)); + StrAppend(res, newsub); start_pos = pos + oldsub.size(); // start searching again after the "old" } while (replace_all); - res->append(s, start_pos, s.length() - start_pos); + StrAppend(res, s.substr(start_pos, s.length() - start_pos)); } // ---------------------------------------------------------------------- @@ -160,8 +159,8 @@ void StringReplace(const std::string &s, const std::string &oldsub, // happened or not. // ---------------------------------------------------------------------- -std::string StringReplace(const std::string &s, const std::string &oldsub, - const std::string &newsub, bool replace_all) { +std::string StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, + bool replace_all) { std::string ret; StringReplace(s, oldsub, newsub, replace_all, &ret); return ret; @@ -445,22 +444,24 @@ int UnescapeCEscapeSequences(const char *source, char *dest, // In the first and second calls, the length of dest is returned. In the // the third call, the new string is returned. // ---------------------------------------------------------------------- -int UnescapeCEscapeString(const std::string &src, std::string *dest) { +int UnescapeCEscapeString(StringPiece src, std::string *dest) { return UnescapeCEscapeString(src, dest, nullptr); } -int UnescapeCEscapeString(const std::string &src, std::string *dest, +int UnescapeCEscapeString(StringPiece src, std::string *dest, std::vector *errors) { std::unique_ptr unescaped(new char[src.size() + 1]); - int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), errors); + int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(), + errors); GOOGLE_CHECK(dest); dest->assign(unescaped.get(), len); return len; } -std::string UnescapeCEscapeString(const std::string &src) { +std::string UnescapeCEscapeString(StringPiece src) { std::unique_ptr unescaped(new char[src.size() + 1]); - int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), nullptr); + int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(), + nullptr); return std::string(unescaped.get(), len); } @@ -592,7 +593,7 @@ void CEscapeAndAppend(StringPiece src, std::string *dest) { } } -std::string CEscape(const std::string &src) { +std::string CEscape(StringPiece src) { std::string dest; CEscapeAndAppend(src, &dest); return dest; @@ -600,7 +601,7 @@ std::string CEscape(const std::string &src) { namespace strings { -std::string Utf8SafeCEscape(const std::string &src) { +std::string Utf8SafeCEscape(StringPiece src) { const int dest_length = src.size() * 4 + 1; // Maximum possible expansion std::unique_ptr dest(new char[dest_length]); const int len = CEscapeInternal(src.data(), src.size(), @@ -609,7 +610,7 @@ std::string Utf8SafeCEscape(const std::string &src) { return std::string(dest.get(), len); } -std::string CHexEscape(const std::string &src) { +std::string CHexEscape(StringPiece src) { const int dest_length = src.size() * 4 + 1; // Maximum possible expansion std::unique_ptr dest(new char[dest_length]); const int len = CEscapeInternal(src.data(), src.size(), @@ -663,7 +664,7 @@ uint32_t strtou32_adaptor(const char *nptr, char **endptr, int base) { return static_cast(result); } -inline bool safe_parse_sign(std::string *text /*inout*/, +inline bool safe_parse_sign(StringPiece *text /*inout*/, bool *negative_ptr /*output*/) { const char* start = text->data(); const char* end = start + text->size(); @@ -692,7 +693,7 @@ inline bool safe_parse_sign(std::string *text /*inout*/, } template -bool safe_parse_positive_int(std::string text, IntType *value_p) { +bool safe_parse_positive_int(StringPiece text, IntType *value_p) { int base = 10; IntType value = 0; const IntType vmax = std::numeric_limits::max(); @@ -725,7 +726,7 @@ bool safe_parse_positive_int(std::string text, IntType *value_p) { } template -bool safe_parse_negative_int(const std::string &text, IntType *value_p) { +bool safe_parse_negative_int(StringPiece text, IntType *value_p) { int base = 10; IntType value = 0; const IntType vmin = std::numeric_limits::min(); @@ -765,7 +766,7 @@ bool safe_parse_negative_int(const std::string &text, IntType *value_p) { } template -bool safe_int_internal(std::string text, IntType *value_p) { +bool safe_int_internal(StringPiece text, IntType *value_p) { *value_p = 0; bool negative; if (!safe_parse_sign(&text, &negative)) { @@ -779,7 +780,7 @@ bool safe_int_internal(std::string text, IntType *value_p) { } template -bool safe_uint_internal(std::string text, IntType *value_p) { +bool safe_uint_internal(StringPiece text, IntType *value_p) { *value_p = 0; bool negative; if (!safe_parse_sign(&text, &negative) || negative) { @@ -1341,19 +1342,19 @@ bool safe_strtod(const char* str, double* value) { return *str != '\0' && *endptr == '\0'; } -bool safe_strto32(const std::string &str, int32_t *value) { +bool safe_strto32(StringPiece str, int32_t *value) { return safe_int_internal(str, value); } -bool safe_strtou32(const std::string &str, uint32_t *value) { +bool safe_strtou32(StringPiece str, uint32_t *value) { return safe_uint_internal(str, value); } -bool safe_strto64(const std::string &str, int64_t *value) { +bool safe_strto64(StringPiece str, int64_t *value) { return safe_int_internal(str, value); } -bool safe_strtou64(const std::string &str, uint64_t *value) { +bool safe_strtou64(StringPiece str, uint64_t *value) { return safe_uint_internal(str, value); } @@ -1611,8 +1612,8 @@ void StrAppend(std::string *result, const AlphaNum &a, const AlphaNum &b, GOOGLE_DCHECK_EQ(out, begin + result->size()); } -int GlobalReplaceSubstring(const std::string &substring, - const std::string &replacement, std::string *s) { +int GlobalReplaceSubstring(StringPiece substring, StringPiece replacement, + std::string *s) { GOOGLE_CHECK(s != nullptr); if (s->empty() || substring.empty()) return 0; @@ -2332,15 +2333,15 @@ int UTF8FirstLetterNumBytes(const char* src, int len) { // (1) determines the presence of LF (first one is ok) // (2) if yes, removes any CR, else convert every CR to LF -void CleanStringLineEndings(const std::string &src, std::string *dst, +void CleanStringLineEndings(StringPiece src, std::string *dst, bool auto_end_last_line) { if (dst->empty()) { - dst->append(src); + StrAppend(dst, src); CleanStringLineEndings(dst, auto_end_last_line); } else { - std::string tmp = src; + std::string tmp(src); CleanStringLineEndings(&tmp, auto_end_last_line); - dst->append(tmp); + StrAppend(dst, tmp); } } diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index 9658abf908..0be4a5eafd 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -118,12 +118,11 @@ inline bool HasPrefixString(StringPiece str, StringPiece prefix) { memcmp(str.data(), prefix.data(), prefix.size()) == 0; } -inline std::string StripPrefixString(const std::string& str, - const std::string& prefix) { +inline std::string StripPrefixString(StringPiece str, StringPiece prefix) { if (HasPrefixString(str, prefix)) { - return str.substr(prefix.size()); + return str.substr(prefix.size()).ToString(); } else { - return str; + return str.ToString(); } } @@ -141,12 +140,11 @@ inline bool HasSuffixString(StringPiece str, StringPiece suffix) { suffix.size()) == 0; } -inline std::string StripSuffixString(const std::string& str, - const std::string& suffix) { +inline std::string StripSuffixString(StringPiece str, StringPiece suffix) { if (HasSuffixString(str, suffix)) { - return str.substr(0, str.size() - suffix.size()); + return str.substr(0, str.size() - suffix.size()).ToString(); } else { - return str; + return str.ToString(); } } @@ -192,8 +190,8 @@ inline void UpperString(std::string* s) { inline void ToUpper(std::string* s) { UpperString(s); } -inline std::string ToUpper(const std::string& s) { - std::string out = s; +inline std::string ToUpper(StringPiece s) { + std::string out(s); UpperString(&out); return out; } @@ -206,10 +204,8 @@ inline std::string ToUpper(const std::string& s) { // happened or not. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT std::string StringReplace(const std::string& s, - const std::string& oldsub, - const std::string& newsub, - bool replace_all); +PROTOBUF_EXPORT std::string StringReplace(StringPiece s, StringPiece oldsub, + StringPiece newsub, bool replace_all); // ---------------------------------------------------------------------- // SplitStringUsing() @@ -314,12 +310,10 @@ PROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, // the third call, the new string is returned. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src, - std::string* dest); -PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src, - std::string* dest, +PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest); +PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest, std::vector* errors); -PROTOBUF_EXPORT std::string UnescapeCEscapeString(const std::string& src); +PROTOBUF_EXPORT std::string UnescapeCEscapeString(StringPiece src); // ---------------------------------------------------------------------- // CEscape() @@ -328,7 +322,7 @@ PROTOBUF_EXPORT std::string UnescapeCEscapeString(const std::string& src); // // Escaped chars: \n, \r, \t, ", ', \, and !isprint(). // ---------------------------------------------------------------------- -PROTOBUF_EXPORT std::string CEscape(const std::string& src); +PROTOBUF_EXPORT std::string CEscape(StringPiece src); // ---------------------------------------------------------------------- // CEscapeAndAppend() @@ -339,10 +333,10 @@ PROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, std::string* dest); namespace strings { // Like CEscape() but does not escape bytes with the upper bit set. -PROTOBUF_EXPORT std::string Utf8SafeCEscape(const std::string& src); +PROTOBUF_EXPORT std::string Utf8SafeCEscape(StringPiece src); // Like CEscape() but uses hex (\x) escapes instead of octals. -PROTOBUF_EXPORT std::string CHexEscape(const std::string& src); +PROTOBUF_EXPORT std::string CHexEscape(StringPiece src); } // namespace strings // ---------------------------------------------------------------------- @@ -399,35 +393,23 @@ inline uint64_t strtou64(const char *nptr, char **endptr, int base) { // ---------------------------------------------------------------------- PROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value); -PROTOBUF_EXPORT bool safe_strto32(const std::string& str, int32_t* value); -PROTOBUF_EXPORT bool safe_strtou32(const std::string& str, uint32_t* value); +PROTOBUF_EXPORT bool safe_strto32(StringPiece str, int32_t* value); +PROTOBUF_EXPORT bool safe_strtou32(StringPiece str, uint32_t* value); inline bool safe_strto32(const char* str, int32_t* value) { return safe_strto32(std::string(str), value); } -inline bool safe_strto32(StringPiece str, int32_t* value) { - return safe_strto32(str.ToString(), value); -} inline bool safe_strtou32(const char* str, uint32_t* value) { return safe_strtou32(std::string(str), value); } -inline bool safe_strtou32(StringPiece str, uint32_t* value) { - return safe_strtou32(str.ToString(), value); -} -PROTOBUF_EXPORT bool safe_strto64(const std::string& str, int64_t* value); -PROTOBUF_EXPORT bool safe_strtou64(const std::string& str, uint64_t* value); +PROTOBUF_EXPORT bool safe_strto64(StringPiece str, int64_t* value); +PROTOBUF_EXPORT bool safe_strtou64(StringPiece str, uint64_t* value); inline bool safe_strto64(const char* str, int64_t* value) { return safe_strto64(std::string(str), value); } -inline bool safe_strto64(StringPiece str, int64_t* value) { - return safe_strto64(str.ToString(), value); -} inline bool safe_strtou64(const char* str, uint64_t* value) { return safe_strtou64(std::string(str), value); } -inline bool safe_strtou64(StringPiece str, uint64_t* value) { - return safe_strtou64(str.ToString(), value); -} PROTOBUF_EXPORT bool safe_strtof(const char* str, float* value); PROTOBUF_EXPORT bool safe_strtod(const char* str, double* value); @@ -798,8 +780,8 @@ PROTOBUF_EXPORT std::string ToHex(uint64_t num); // // NOTE: The string pieces must not overlap s. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT int GlobalReplaceSubstring(const std::string& substring, - const std::string& replacement, +PROTOBUF_EXPORT int GlobalReplaceSubstring(StringPiece substring, + StringPiece replacement, std::string* s); // ---------------------------------------------------------------------- @@ -917,8 +899,7 @@ PROTOBUF_EXPORT int UTF8FirstLetterNumBytes(const char* src, int len); // // (1) determines the presence of LF (first one is ok) // (2) if yes, removes any CR, else convert every CR to LF -PROTOBUF_EXPORT void CleanStringLineEndings(const std::string& src, - std::string* dst, +PROTOBUF_EXPORT void CleanStringLineEndings(StringPiece src, std::string* dst, bool auto_end_last_line); // Same as above, but transforms the argument in place. diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index afa5e2e474..777d060bbd 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -73,7 +73,6 @@ ProtoWriter::ProtoWriter(TypeResolver* type_resolver, element_(nullptr), size_insert_(), output_(output), - buffer_(), adapter_(&buffer_), stream_(new CodedOutputStream(&adapter_)), listener_(listener), @@ -95,7 +94,6 @@ ProtoWriter::ProtoWriter(const TypeInfo* typeinfo, element_(nullptr), size_insert_(), output_(output), - buffer_(), adapter_(&buffer_), stream_(new CodedOutputStream(&adapter_)), listener_(listener), diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index a8b68a3c76..29ffe865d9 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -84,7 +84,7 @@ util::Status GetStatus(const util::StatusOr& s) { } MATCHER_P(StatusIs, status, - StrCat(".status() is ", testing::PrintToString(status))) { + StrCat(".status() is ", testing::PrintToString(status))) { return GetStatus(arg).code() == status; } @@ -412,6 +412,110 @@ TEST(JsonUtilTest, TestDynamicMessage) { EXPECT_EQ(*message_json, *generated_json); } +TEST(JsonUtilTest, TestParsingAny) { + StringPiece input = R"json( + { + "value": { + "@type": "type.googleapis.com/proto3.TestMessage", + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1} + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); + + EXPECT_THAT( + ToJson(m), + IsOkAndHolds( + R"json({"value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}})json")); +} + +TEST(JsonUtilTest, TestParsingAnyMiddleAtType) { + StringPiece input = R"json( + { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "@type": "type.googleapis.com/proto3.TestMessage", + "message_value": {"value": 1} + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); +} + +TEST(JsonUtilTest, TestParsingAnyEndAtType) { + StringPiece input = R"json( + { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1}, + "@type": "type.googleapis.com/proto3.TestMessage" + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); +} + +TEST(JsonUtilTest, TestParsingNestedAnys) { + StringPiece input = R"json( + { + "value": { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1}, + "@type": "type.googleapis.com/proto3.TestMessage" + }, + "@type": "type.googleapis.com/google.protobuf.Any" + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + google::protobuf::Any inner; + EXPECT_TRUE(m.value().UnpackTo(&inner)); + + TestMessage t; + EXPECT_TRUE(inner.UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); + + EXPECT_THAT( + ToJson(m), + IsOkAndHolds( + R"json({"value":{"@type":"type.googleapis.com/google.protobuf.Any","value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})json")); +} + TEST(JsonUtilTest, TestParsingUnknownAnyFields) { StringPiece input = R"json( { @@ -672,9 +776,9 @@ TEST(JsonUtilTest, TestWrongJsonInput) { auto* resolver = NewTypeResolverForDescriptorPool( "type.googleapis.com", DescriptorPool::generated_pool()); - EXPECT_THAT(JsonToBinaryStream(resolver, message_type, &input_stream, - &output_stream), - StatusIs(util::StatusCode::kInvalidArgument)); + EXPECT_THAT( + JsonToBinaryStream(resolver, message_type, &input_stream, &output_stream), + StatusIs(util::StatusCode::kInvalidArgument)); delete resolver; } From 8deb95bda97ec599c375c366057de00d1b1c54a8 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Fri, 17 Jun 2022 13:59:28 -0700 Subject: [PATCH 02/11] Fix typo and update CHANGES.txt --- CHANGES.txt | 4 ++++ conformance/conformance_cpp.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 12360dadc6..c3124d73cb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,10 @@ Unreleased version * Handle reflection for message splitting. * make metadata fields lazy. * Extend visibility of plugin library to upb + * Take StringPiece instead of const string& in absl stubs. + * Modernize conformance_cpp.cc. + * Don't request 64-byte alignment unless the toolchain supports it. + 2022-05-27 version 21.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby) diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 1185880715..3181a54c67 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -270,6 +270,6 @@ int main() { } total_runs++; } - GOOGLE_LOG(INFO) << "conformance-cpp: recieved EOF from test runner after " + GOOGLE_LOG(INFO) << "conformance-cpp: received EOF from test runner after " << total_runs << " tests"; } From a94d41960ef07eabb5598deb1ce7015c1435312b Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Fri, 17 Jun 2022 17:19:14 -0700 Subject: [PATCH 03/11] Add missing file names to BUILD.bazel files. --- src/google/protobuf/compiler/java/BUILD.bazel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 619d71518a..5e61898c40 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -31,6 +31,7 @@ cc_library( "message_field.cc", "message_field_lite.cc", "message_lite.cc", + "message_serialization.cc", "name_resolver.cc", "primitive_field.cc", "primitive_field_lite.cc", @@ -62,6 +63,7 @@ cc_library( "message_field.h", "message_field_lite.h", "message_lite.h", + "message_serialization.h", "name_resolver.h", "names.h", "options.h", From 126841ed7d3caad19e0a8de624689dd2978d90b2 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Fri, 17 Jun 2022 19:34:14 -0700 Subject: [PATCH 04/11] Add some extra tracing to figure out a crash during build --- src/google/protobuf/stubs/strutil.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 23a76d0599..95fb25434c 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -129,9 +129,14 @@ void StripWhitespace(std::string *str) { // it only replaces the first instance of "old." // ---------------------------------------------------------------------- +#define DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(dest, src) \ + GOOGLE_DCHECK_GT(uintptr_t((src).data() - (dest).data()), \ + uintptr_t((dest).size())) + void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, bool replace_all, std::string *res) { if (oldsub.empty()) { + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); StrAppend(res, s); // if empty, append the given string. return; } @@ -143,10 +148,13 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, if (pos == std::string::npos) { break; } + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); StrAppend(res, s.substr(start_pos, pos - start_pos)); + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, newsub); StrAppend(res, newsub); start_pos = pos + oldsub.size(); // start searching again after the "old" } while (replace_all); + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); StrAppend(res, s.substr(start_pos, s.length() - start_pos)); } From dc7f4debbc2f06b2940a0df8a3baa0c9fb336c39 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Fri, 17 Jun 2022 21:11:17 -0700 Subject: [PATCH 05/11] More debugging information please --- src/google/protobuf/stubs/strutil.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 95fb25434c..767b6342c0 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -154,7 +154,7 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, StrAppend(res, newsub); start_pos = pos + oldsub.size(); // start searching again after the "old" } while (replace_all); - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s) << " res=" << *res << " s=" << s; StrAppend(res, s.substr(start_pos, s.length() - start_pos)); } From e9668e41be9aa7bc2d26406ecec08908f905ef49 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 14:54:10 -0700 Subject: [PATCH 06/11] Find out who is calling StringReplace, or if the call is from strutil itself. --- src/google/protobuf/stubs/strutil.cc | 13 ++++++++----- src/google/protobuf/stubs/strutil.h | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 767b6342c0..e5d0078abd 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -43,6 +43,8 @@ #include #include +#undef StringReplace + #ifdef _WIN32 // MSVC has only _snprintf, not snprintf. // @@ -134,7 +136,7 @@ void StripWhitespace(std::string *str) { uintptr_t((dest).size())) void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all, std::string *res) { + bool replace_all, std::string *res, char *filename, int linenum) { if (oldsub.empty()) { DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); StrAppend(res, s); // if empty, append the given string. @@ -154,7 +156,8 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, StrAppend(res, newsub); start_pos = pos + oldsub.size(); // start searching again after the "old" } while (replace_all); - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s) << " res=" << *res << " s=" << s; + DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s) + << " file=" << (filename ? filename : "") << " line=" << linenum; StrAppend(res, s.substr(start_pos, s.length() - start_pos)); } @@ -167,10 +170,10 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, // happened or not. // ---------------------------------------------------------------------- -std::string StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all) { +std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, StringPiece newsub, + bool replace_all, char *filename, int linenum) { std::string ret; - StringReplace(s, oldsub, newsub, replace_all, &ret); + StringReplace(s, oldsub, newsub, replace_all, &ret, filename, linenum); return ret; } diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index 0be4a5eafd..541fa9f99b 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -204,8 +204,9 @@ inline std::string ToUpper(StringPiece s) { // happened or not. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT std::string StringReplace(StringPiece s, StringPiece oldsub, - StringPiece newsub, bool replace_all); +PROTOBUF_EXPORT std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, + StringPiece newsub, bool replace_all, char *filename = nullptr, int linenum = 0); +#define StringReplace(s,o,n,all) StringReplaceImpl(s,o,n,all,__FILE__, __LINE__); // ---------------------------------------------------------------------- // SplitStringUsing() From 81491e034ff561c7ce89bad60dbbe49ccc4fd40b Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 15:16:40 -0700 Subject: [PATCH 07/11] Can't use char * for quoted literals. I knew that. --- src/google/protobuf/stubs/strutil.cc | 4 ++-- src/google/protobuf/stubs/strutil.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index e5d0078abd..fe2b07b194 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -136,7 +136,7 @@ void StripWhitespace(std::string *str) { uintptr_t((dest).size())) void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all, std::string *res, char *filename, int linenum) { + bool replace_all, std::string *res, const char *filename, int linenum) { if (oldsub.empty()) { DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); StrAppend(res, s); // if empty, append the given string. @@ -171,7 +171,7 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, // ---------------------------------------------------------------------- std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all, char *filename, int linenum) { + bool replace_all, const char *filename, int linenum) { std::string ret; StringReplace(s, oldsub, newsub, replace_all, &ret, filename, linenum); return ret; diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index 541fa9f99b..8657df2cf2 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -205,7 +205,7 @@ inline std::string ToUpper(StringPiece s) { // ---------------------------------------------------------------------- PROTOBUF_EXPORT std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, - StringPiece newsub, bool replace_all, char *filename = nullptr, int linenum = 0); + StringPiece newsub, bool replace_all, const char *filename = nullptr, int linenum = 0); #define StringReplace(s,o,n,all) StringReplaceImpl(s,o,n,all,__FILE__, __LINE__); // ---------------------------------------------------------------------- From a075d2fb14ac2efa8076ae0d60449257288ddc21 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 15:27:10 -0700 Subject: [PATCH 08/11] Remove extraneous ';' --- src/google/protobuf/stubs/strutil.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index 8657df2cf2..3eeb727c32 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -206,7 +206,7 @@ inline std::string ToUpper(StringPiece s) { PROTOBUF_EXPORT std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, StringPiece newsub, bool replace_all, const char *filename = nullptr, int linenum = 0); -#define StringReplace(s,o,n,all) StringReplaceImpl(s,o,n,all,__FILE__, __LINE__); +#define StringReplace(s,o,n,all) StringReplaceImpl(s,o,n,all,__FILE__, __LINE__) // ---------------------------------------------------------------------- // SplitStringUsing() From 24d0a3bed154c46d3bb3f9138540e7b237f98bb2 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 18:01:06 -0700 Subject: [PATCH 09/11] Revert all changes to strutil.{h,cc} from this PR --- src/google/protobuf/stubs/strutil.cc | 84 ++++++++++++---------------- src/google/protobuf/stubs/strutil.h | 66 ++++++++++++++-------- 2 files changed, 77 insertions(+), 73 deletions(-) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index fe2b07b194..594c8eac6a 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -43,8 +43,6 @@ #include #include -#undef StringReplace - #ifdef _WIN32 // MSVC has only _snprintf, not snprintf. // @@ -131,15 +129,11 @@ void StripWhitespace(std::string *str) { // it only replaces the first instance of "old." // ---------------------------------------------------------------------- -#define DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(dest, src) \ - GOOGLE_DCHECK_GT(uintptr_t((src).data() - (dest).data()), \ - uintptr_t((dest).size())) - -void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all, std::string *res, const char *filename, int linenum) { +void StringReplace(const std::string &s, const std::string &oldsub, + const std::string &newsub, bool replace_all, + std::string *res) { if (oldsub.empty()) { - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); - StrAppend(res, s); // if empty, append the given string. + res->append(s); // if empty, append the given string. return; } @@ -150,15 +144,11 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, if (pos == std::string::npos) { break; } - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s); - StrAppend(res, s.substr(start_pos, pos - start_pos)); - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, newsub); - StrAppend(res, newsub); + res->append(s, start_pos, pos - start_pos); + res->append(newsub); start_pos = pos + oldsub.size(); // start searching again after the "old" } while (replace_all); - DONOTCHECKIN_GOOGLE_DCHECK_NO_OVERLAP(*res, s) - << " file=" << (filename ? filename : "") << " line=" << linenum; - StrAppend(res, s.substr(start_pos, s.length() - start_pos)); + res->append(s, start_pos, s.length() - start_pos); } // ---------------------------------------------------------------------- @@ -170,10 +160,10 @@ void StringReplace(StringPiece s, StringPiece oldsub, StringPiece newsub, // happened or not. // ---------------------------------------------------------------------- -std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, StringPiece newsub, - bool replace_all, const char *filename, int linenum) { +std::string StringReplace(const std::string &s, const std::string &oldsub, + const std::string &newsub, bool replace_all) { std::string ret; - StringReplace(s, oldsub, newsub, replace_all, &ret, filename, linenum); + StringReplace(s, oldsub, newsub, replace_all, &ret); return ret; } @@ -455,24 +445,22 @@ int UnescapeCEscapeSequences(const char *source, char *dest, // In the first and second calls, the length of dest is returned. In the // the third call, the new string is returned. // ---------------------------------------------------------------------- -int UnescapeCEscapeString(StringPiece src, std::string *dest) { +int UnescapeCEscapeString(const std::string &src, std::string *dest) { return UnescapeCEscapeString(src, dest, nullptr); } -int UnescapeCEscapeString(StringPiece src, std::string *dest, +int UnescapeCEscapeString(const std::string &src, std::string *dest, std::vector *errors) { std::unique_ptr unescaped(new char[src.size() + 1]); - int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(), - errors); + int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), errors); GOOGLE_CHECK(dest); dest->assign(unescaped.get(), len); return len; } -std::string UnescapeCEscapeString(StringPiece src) { +std::string UnescapeCEscapeString(const std::string &src) { std::unique_ptr unescaped(new char[src.size() + 1]); - int len = UnescapeCEscapeSequences(std::string(src).c_str(), unescaped.get(), - nullptr); + int len = UnescapeCEscapeSequences(src.c_str(), unescaped.get(), nullptr); return std::string(unescaped.get(), len); } @@ -513,11 +501,9 @@ int CEscapeInternal(const char* src, int src_len, char* dest, if ((!utf8_safe || static_cast(*src) < 0x80) && (!isprint(*src) || (last_hex_escape && isxdigit(*src)))) { - // need space for 4 letter escape and the trailing '\0' to - // be written by snprintf. - if (dest_len - used < 5) + if (dest_len - used < 4) // need space for 4 letter escape return -1; - snprintf(dest + used, 5, (use_hex ? "\\x%02x" : "\\%03o"), + sprintf(dest + used, (use_hex ? "\\x%02x" : "\\%03o"), static_cast(*src)); is_hex_escape = use_hex; used += 4; @@ -606,7 +592,7 @@ void CEscapeAndAppend(StringPiece src, std::string *dest) { } } -std::string CEscape(StringPiece src) { +std::string CEscape(const std::string &src) { std::string dest; CEscapeAndAppend(src, &dest); return dest; @@ -614,7 +600,7 @@ std::string CEscape(StringPiece src) { namespace strings { -std::string Utf8SafeCEscape(StringPiece src) { +std::string Utf8SafeCEscape(const std::string &src) { const int dest_length = src.size() * 4 + 1; // Maximum possible expansion std::unique_ptr dest(new char[dest_length]); const int len = CEscapeInternal(src.data(), src.size(), @@ -623,7 +609,7 @@ std::string Utf8SafeCEscape(StringPiece src) { return std::string(dest.get(), len); } -std::string CHexEscape(StringPiece src) { +std::string CHexEscape(const std::string &src) { const int dest_length = src.size() * 4 + 1; // Maximum possible expansion std::unique_ptr dest(new char[dest_length]); const int len = CEscapeInternal(src.data(), src.size(), @@ -677,7 +663,7 @@ uint32_t strtou32_adaptor(const char *nptr, char **endptr, int base) { return static_cast(result); } -inline bool safe_parse_sign(StringPiece *text /*inout*/, +inline bool safe_parse_sign(std::string *text /*inout*/, bool *negative_ptr /*output*/) { const char* start = text->data(); const char* end = start + text->size(); @@ -706,7 +692,7 @@ inline bool safe_parse_sign(StringPiece *text /*inout*/, } template -bool safe_parse_positive_int(StringPiece text, IntType *value_p) { +bool safe_parse_positive_int(std::string text, IntType *value_p) { int base = 10; IntType value = 0; const IntType vmax = std::numeric_limits::max(); @@ -739,7 +725,7 @@ bool safe_parse_positive_int(StringPiece text, IntType *value_p) { } template -bool safe_parse_negative_int(StringPiece text, IntType *value_p) { +bool safe_parse_negative_int(const std::string &text, IntType *value_p) { int base = 10; IntType value = 0; const IntType vmin = std::numeric_limits::min(); @@ -779,7 +765,7 @@ bool safe_parse_negative_int(StringPiece text, IntType *value_p) { } template -bool safe_int_internal(StringPiece text, IntType *value_p) { +bool safe_int_internal(std::string text, IntType *value_p) { *value_p = 0; bool negative; if (!safe_parse_sign(&text, &negative)) { @@ -793,7 +779,7 @@ bool safe_int_internal(StringPiece text, IntType *value_p) { } template -bool safe_uint_internal(StringPiece text, IntType *value_p) { +bool safe_uint_internal(std::string text, IntType *value_p) { *value_p = 0; bool negative; if (!safe_parse_sign(&text, &negative) || negative) { @@ -1355,19 +1341,19 @@ bool safe_strtod(const char* str, double* value) { return *str != '\0' && *endptr == '\0'; } -bool safe_strto32(StringPiece str, int32_t *value) { +bool safe_strto32(const std::string &str, int32_t *value) { return safe_int_internal(str, value); } -bool safe_strtou32(StringPiece str, uint32_t *value) { +bool safe_strtou32(const std::string &str, uint32_t *value) { return safe_uint_internal(str, value); } -bool safe_strto64(StringPiece str, int64_t *value) { +bool safe_strto64(const std::string &str, int64_t *value) { return safe_int_internal(str, value); } -bool safe_strtou64(StringPiece str, uint64_t *value) { +bool safe_strtou64(const std::string &str, uint64_t *value) { return safe_uint_internal(str, value); } @@ -1625,8 +1611,8 @@ void StrAppend(std::string *result, const AlphaNum &a, const AlphaNum &b, GOOGLE_DCHECK_EQ(out, begin + result->size()); } -int GlobalReplaceSubstring(StringPiece substring, StringPiece replacement, - std::string *s) { +int GlobalReplaceSubstring(const std::string &substring, + const std::string &replacement, std::string *s) { GOOGLE_CHECK(s != nullptr); if (s->empty() || substring.empty()) return 0; @@ -2346,15 +2332,15 @@ int UTF8FirstLetterNumBytes(const char* src, int len) { // (1) determines the presence of LF (first one is ok) // (2) if yes, removes any CR, else convert every CR to LF -void CleanStringLineEndings(StringPiece src, std::string *dst, +void CleanStringLineEndings(const std::string &src, std::string *dst, bool auto_end_last_line) { if (dst->empty()) { - StrAppend(dst, src); + dst->append(src); CleanStringLineEndings(dst, auto_end_last_line); } else { - std::string tmp(src); + std::string tmp = src; CleanStringLineEndings(&tmp, auto_end_last_line); - StrAppend(dst, tmp); + dst->append(tmp); } } diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index 3eeb727c32..9658abf908 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -118,11 +118,12 @@ inline bool HasPrefixString(StringPiece str, StringPiece prefix) { memcmp(str.data(), prefix.data(), prefix.size()) == 0; } -inline std::string StripPrefixString(StringPiece str, StringPiece prefix) { +inline std::string StripPrefixString(const std::string& str, + const std::string& prefix) { if (HasPrefixString(str, prefix)) { - return str.substr(prefix.size()).ToString(); + return str.substr(prefix.size()); } else { - return str.ToString(); + return str; } } @@ -140,11 +141,12 @@ inline bool HasSuffixString(StringPiece str, StringPiece suffix) { suffix.size()) == 0; } -inline std::string StripSuffixString(StringPiece str, StringPiece suffix) { +inline std::string StripSuffixString(const std::string& str, + const std::string& suffix) { if (HasSuffixString(str, suffix)) { - return str.substr(0, str.size() - suffix.size()).ToString(); + return str.substr(0, str.size() - suffix.size()); } else { - return str.ToString(); + return str; } } @@ -190,8 +192,8 @@ inline void UpperString(std::string* s) { inline void ToUpper(std::string* s) { UpperString(s); } -inline std::string ToUpper(StringPiece s) { - std::string out(s); +inline std::string ToUpper(const std::string& s) { + std::string out = s; UpperString(&out); return out; } @@ -204,9 +206,10 @@ inline std::string ToUpper(StringPiece s) { // happened or not. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT std::string StringReplaceImpl(StringPiece s, StringPiece oldsub, - StringPiece newsub, bool replace_all, const char *filename = nullptr, int linenum = 0); -#define StringReplace(s,o,n,all) StringReplaceImpl(s,o,n,all,__FILE__, __LINE__) +PROTOBUF_EXPORT std::string StringReplace(const std::string& s, + const std::string& oldsub, + const std::string& newsub, + bool replace_all); // ---------------------------------------------------------------------- // SplitStringUsing() @@ -311,10 +314,12 @@ PROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, // the third call, the new string is returned. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest); -PROTOBUF_EXPORT int UnescapeCEscapeString(StringPiece src, std::string* dest, +PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src, + std::string* dest); +PROTOBUF_EXPORT int UnescapeCEscapeString(const std::string& src, + std::string* dest, std::vector* errors); -PROTOBUF_EXPORT std::string UnescapeCEscapeString(StringPiece src); +PROTOBUF_EXPORT std::string UnescapeCEscapeString(const std::string& src); // ---------------------------------------------------------------------- // CEscape() @@ -323,7 +328,7 @@ PROTOBUF_EXPORT std::string UnescapeCEscapeString(StringPiece src); // // Escaped chars: \n, \r, \t, ", ', \, and !isprint(). // ---------------------------------------------------------------------- -PROTOBUF_EXPORT std::string CEscape(StringPiece src); +PROTOBUF_EXPORT std::string CEscape(const std::string& src); // ---------------------------------------------------------------------- // CEscapeAndAppend() @@ -334,10 +339,10 @@ PROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, std::string* dest); namespace strings { // Like CEscape() but does not escape bytes with the upper bit set. -PROTOBUF_EXPORT std::string Utf8SafeCEscape(StringPiece src); +PROTOBUF_EXPORT std::string Utf8SafeCEscape(const std::string& src); // Like CEscape() but uses hex (\x) escapes instead of octals. -PROTOBUF_EXPORT std::string CHexEscape(StringPiece src); +PROTOBUF_EXPORT std::string CHexEscape(const std::string& src); } // namespace strings // ---------------------------------------------------------------------- @@ -394,23 +399,35 @@ inline uint64_t strtou64(const char *nptr, char **endptr, int base) { // ---------------------------------------------------------------------- PROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value); -PROTOBUF_EXPORT bool safe_strto32(StringPiece str, int32_t* value); -PROTOBUF_EXPORT bool safe_strtou32(StringPiece str, uint32_t* value); +PROTOBUF_EXPORT bool safe_strto32(const std::string& str, int32_t* value); +PROTOBUF_EXPORT bool safe_strtou32(const std::string& str, uint32_t* value); inline bool safe_strto32(const char* str, int32_t* value) { return safe_strto32(std::string(str), value); } +inline bool safe_strto32(StringPiece str, int32_t* value) { + return safe_strto32(str.ToString(), value); +} inline bool safe_strtou32(const char* str, uint32_t* value) { return safe_strtou32(std::string(str), value); } +inline bool safe_strtou32(StringPiece str, uint32_t* value) { + return safe_strtou32(str.ToString(), value); +} -PROTOBUF_EXPORT bool safe_strto64(StringPiece str, int64_t* value); -PROTOBUF_EXPORT bool safe_strtou64(StringPiece str, uint64_t* value); +PROTOBUF_EXPORT bool safe_strto64(const std::string& str, int64_t* value); +PROTOBUF_EXPORT bool safe_strtou64(const std::string& str, uint64_t* value); inline bool safe_strto64(const char* str, int64_t* value) { return safe_strto64(std::string(str), value); } +inline bool safe_strto64(StringPiece str, int64_t* value) { + return safe_strto64(str.ToString(), value); +} inline bool safe_strtou64(const char* str, uint64_t* value) { return safe_strtou64(std::string(str), value); } +inline bool safe_strtou64(StringPiece str, uint64_t* value) { + return safe_strtou64(str.ToString(), value); +} PROTOBUF_EXPORT bool safe_strtof(const char* str, float* value); PROTOBUF_EXPORT bool safe_strtod(const char* str, double* value); @@ -781,8 +798,8 @@ PROTOBUF_EXPORT std::string ToHex(uint64_t num); // // NOTE: The string pieces must not overlap s. // ---------------------------------------------------------------------- -PROTOBUF_EXPORT int GlobalReplaceSubstring(StringPiece substring, - StringPiece replacement, +PROTOBUF_EXPORT int GlobalReplaceSubstring(const std::string& substring, + const std::string& replacement, std::string* s); // ---------------------------------------------------------------------- @@ -900,7 +917,8 @@ PROTOBUF_EXPORT int UTF8FirstLetterNumBytes(const char* src, int len); // // (1) determines the presence of LF (first one is ok) // (2) if yes, removes any CR, else convert every CR to LF -PROTOBUF_EXPORT void CleanStringLineEndings(StringPiece src, std::string* dst, +PROTOBUF_EXPORT void CleanStringLineEndings(const std::string& src, + std::string* dst, bool auto_end_last_line); // Same as above, but transforms the argument in place. From 7bb756c1cc580380c689a9141b27e20c7a5d95f3 Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 19:22:47 -0700 Subject: [PATCH 10/11] Re-instate commit c0fc2e881bc36aafb0bf539bf41889611370f60c --- src/google/protobuf/stubs/strutil.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 594c8eac6a..8e9b398360 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -501,9 +501,11 @@ int CEscapeInternal(const char* src, int src_len, char* dest, if ((!utf8_safe || static_cast(*src) < 0x80) && (!isprint(*src) || (last_hex_escape && isxdigit(*src)))) { - if (dest_len - used < 4) // need space for 4 letter escape + // need space for 4 letter escape and the trailing '\0' to + // be written by snprintf. + if (dest_len - used < 5) return -1; - sprintf(dest + used, (use_hex ? "\\x%02x" : "\\%03o"), + snprintf(dest + used, 5, (use_hex ? "\\x%02x" : "\\%03o"), static_cast(*src)); is_hex_escape = use_hex; used += 4; From feb726159f7332ddd24ac77ecb6aa92b8860d65c Mon Sep 17 00:00:00 2001 From: Jorg Brown Date: Sat, 18 Jun 2022 19:27:22 -0700 Subject: [PATCH 11/11] Remove `* Take StringPiece instead of const string& in absl stubs.` from CHANGES.txt since we're not taking it. --- CHANGES.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index c3124d73cb..e6593c72a3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,7 +3,6 @@ Unreleased version * Handle reflection for message splitting. * make metadata fields lazy. * Extend visibility of plugin library to upb - * Take StringPiece instead of const string& in absl stubs. * Modernize conformance_cpp.cc. * Don't request 64-byte alignment unless the toolchain supports it.