diff --git a/CHANGES.txt b/CHANGES.txt index 12360dadc6..e6593c72a3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,9 @@ Unreleased version * Handle reflection for message splitting. * make metadata fields lazy. * Extend visibility of plugin library to upb + * Modernize conformance_cpp.cc. + * Don't request 64-byte alignment unless the toolchain supports it. + 2022-05-27 version 21.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby) diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 5782789dfd..3181a54c67 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -32,165 +32,176 @@ #include #include +#include +#include +#include + +#include +#include #include #include #include #include #include +#include +#include "conformance.pb.h" #include "conformance.pb.h" #include #include +#include +#include #include +#include -using conformance::ConformanceRequest; -using conformance::ConformanceResponse; -using google::protobuf::Descriptor; -using google::protobuf::DescriptorPool; -using google::protobuf::Message; -using google::protobuf::MessageFactory; -using google::protobuf::TextFormat; -using google::protobuf::util::BinaryToJsonString; -using google::protobuf::util::JsonParseOptions; -using google::protobuf::util::JsonToBinaryString; -using google::protobuf::util::NewTypeResolverForDescriptorPool; -using google::protobuf::util::TypeResolver; -using protobuf_test_messages::proto3::TestAllTypesProto3; -using protobuf_test_messages::proto2::TestAllTypesProto2; -using std::string; - -static const char kTypeUrlPrefix[] = "type.googleapis.com"; - -const char* kFailures[] = { -}; - -static string GetTypeUrl(const Descriptor* message) { - return string(kTypeUrlPrefix) + "/" + message->full_name(); -} - -int test_count = 0; -bool verbose = false; -TypeResolver* type_resolver; -string* type_url; +// Must be included last. +#include namespace google { namespace protobuf { - -using util::Status; - -bool CheckedRead(int fd, void *buf, size_t len) { - size_t ofs = 0; +namespace { +using ::conformance::ConformanceRequest; +using ::conformance::ConformanceResponse; +using ::google::protobuf::util::BinaryToJsonString; +using ::google::protobuf::util::JsonParseOptions; +using ::google::protobuf::util::JsonToBinaryString; +using ::google::protobuf::util::NewTypeResolverForDescriptorPool; +using ::google::protobuf::util::TypeResolver; +using ::protobuf_test_messages::proto2::TestAllTypesProto2; +using ::protobuf_test_messages::proto3::TestAllTypesProto3; + +util::Status ReadFd(int fd, char* buf, size_t len) { while (len > 0) { - ssize_t bytes_read = read(fd, (char*)buf + ofs, len); + ssize_t bytes_read = read(fd, buf, len); - if (bytes_read == 0) return false; + if (bytes_read == 0) { + return util::DataLossError("unexpected EOF"); + } if (bytes_read < 0) { - GOOGLE_LOG(FATAL) << "Error reading from test runner: " << strerror(errno); + return util::ErrnoToStatus(errno, "error reading from test runner"); } len -= bytes_read; - ofs += bytes_read; + buf += bytes_read; } - - return true; + return util::OkStatus(); } -void CheckedWrite(int fd, const void *buf, size_t len) { +util::Status WriteFd(int fd, const void* buf, size_t len) { if (write(fd, buf, len) != len) { - GOOGLE_LOG(FATAL) << "Error writing to test runner: " << strerror(errno); + return util::ErrnoToStatus(errno, "error reading to test runner"); } + return util::OkStatus(); } -void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { - Message *test_message; - google::protobuf::LinkMessageReflection(); - google::protobuf::LinkMessageReflection(); - const Descriptor *descriptor = DescriptorPool::generated_pool()->FindMessageTypeByName( - request.message_type()); - if (!descriptor) { - GOOGLE_LOG(FATAL) << "No such message type: " << request.message_type(); +class Harness { + public: + Harness() { + google::protobuf::LinkMessageReflection(); + google::protobuf::LinkMessageReflection(); + + resolver_.reset(NewTypeResolverForDescriptorPool( + "type.googleapis.com", DescriptorPool::generated_pool())); + type_url_ = StrCat("type.googleapis.com/", + TestAllTypesProto3::GetDescriptor()->full_name()); + } + + util::StatusOr RunTest( + const ConformanceRequest& request); + + // Returns Ok(true) if we're done processing requests. + util::StatusOr ServeConformanceRequest(); + + private: + bool verbose_ = false; + std::unique_ptr resolver_; + std::string type_url_; +}; + +util::StatusOr Harness::RunTest( + const ConformanceRequest& request) { + const Descriptor* descriptor = + DescriptorPool::generated_pool()->FindMessageTypeByName( + request.message_type()); + if (descriptor == nullptr) { + return util::NotFoundError( + StrCat("No such message type: ", request.message_type())); } - test_message = MessageFactory::generated_factory()->GetPrototype(descriptor)->New(); + + std::unique_ptr test_message( + MessageFactory::generated_factory()->GetPrototype(descriptor)->New()); + ConformanceResponse response; switch (request.payload_case()) { case ConformanceRequest::kProtobufPayload: { if (!test_message->ParseFromString(request.protobuf_payload())) { - // Getting parse details would involve something like: - // http://stackoverflow.com/questions/22121922/how-can-i-get-more-details-about-errors-generated-during-protobuf-parsing-c - response->set_parse_error("Parse error (no more details available)."); - return; + response.set_parse_error("parse error (no more details available)"); + return response; } break; } case ConformanceRequest::kJsonPayload: { - string proto_binary; JsonParseOptions options; options.ignore_unknown_fields = (request.test_category() == - conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); + + std::string proto_binary; util::Status status = - JsonToBinaryString(type_resolver, *type_url, request.json_payload(), + JsonToBinaryString(resolver_.get(), type_url_, request.json_payload(), &proto_binary, options); if (!status.ok()) { - response->set_parse_error(string("Parse error: ") + - std::string(status.message())); - return; + response.set_parse_error( + StrCat("parse error: ", status.message())); + return response; } if (!test_message->ParseFromString(proto_binary)) { - response->set_runtime_error( - "Parsing JSON generates invalid proto output."); - return; + response.set_runtime_error( + "parsing JSON generated invalid proto output"); + return response; } + break; } case ConformanceRequest::kTextPayload: { - if (!TextFormat::ParseFromString(request.text_payload(), test_message)) { - response->set_parse_error("Parse error"); - return; + if (!TextFormat::ParseFromString(request.text_payload(), + test_message.get())) { + response.set_parse_error("parse error (no more details available)"); + return response; } break; } case ConformanceRequest::PAYLOAD_NOT_SET: - GOOGLE_LOG(FATAL) << "Request didn't have payload."; - break; + return util::InvalidArgumentError("request didn't have payload"); default: - GOOGLE_LOG(FATAL) << "unknown payload type: " << request.payload_case(); - break; - } - - conformance::FailureSet failures; - if (descriptor == failures.GetDescriptor()) { - for (const char* s : kFailures) failures.add_failure(s); - test_message = &failures; + return util::InvalidArgumentError( + StrCat("unknown payload type", request.payload_case())); } switch (request.requested_output_format()) { case conformance::UNSPECIFIED: - GOOGLE_LOG(FATAL) << "Unspecified output format"; - break; + return util::InvalidArgumentError("unspecified output format"); case conformance::PROTOBUF: { - GOOGLE_CHECK(test_message->SerializeToString( - response->mutable_protobuf_payload())); + GOOGLE_CHECK( + test_message->SerializeToString(response.mutable_protobuf_payload())); break; } case conformance::JSON: { - string proto_binary; + std::string proto_binary; GOOGLE_CHECK(test_message->SerializeToString(&proto_binary)); util::Status status = - BinaryToJsonString(type_resolver, *type_url, proto_binary, - response->mutable_json_payload()); + BinaryToJsonString(resolver_.get(), type_url_, proto_binary, + response.mutable_json_payload()); if (!status.ok()) { - response->set_serialize_error( - string("Failed to serialize JSON output: ") + - std::string(status.message())); - return; + response.set_serialize_error(StrCat( + "failed to serialize JSON output: ", status.message())); } break; } @@ -199,70 +210,66 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { TextFormat::Printer printer; printer.SetHideUnknownFields(!request.print_unknown_fields()); GOOGLE_CHECK(printer.PrintToString(*test_message, - response->mutable_text_payload())); + response.mutable_text_payload())); break; } default: - GOOGLE_LOG(FATAL) << "Unknown output format: " - << request.requested_output_format(); + return util::InvalidArgumentError(StrCat( + "unknown output format", request.requested_output_format())); } -} -bool DoTestIo() { - string serialized_input; - string serialized_output; - ConformanceRequest request; - ConformanceResponse response; - uint32_t bytes; + return response; +} - if (!CheckedRead(STDIN_FILENO, &bytes, sizeof(uint32_t))) { - // EOF. - return false; +util::StatusOr Harness::ServeConformanceRequest() { + uint32_t in_len; + if (!ReadFd(STDIN_FILENO, reinterpret_cast(&in_len), sizeof(in_len)) + .ok()) { + // EOF means we're done. + return true; } - serialized_input.resize(bytes); - - if (!CheckedRead(STDIN_FILENO, (char*)serialized_input.c_str(), bytes)) { - GOOGLE_LOG(ERROR) << "Unexpected EOF on stdin. " << strerror(errno); - } + std::string serialized_input; + serialized_input.resize(in_len); + RETURN_IF_ERROR(ReadFd(STDIN_FILENO, serialized_input.data(), in_len)); - if (!request.ParseFromString(serialized_input)) { - GOOGLE_LOG(FATAL) << "Parse of ConformanceRequest proto failed."; - return false; - } + ConformanceRequest request; + GOOGLE_CHECK(request.ParseFromString(serialized_input)); - DoTest(request, &response); + util::StatusOr response = RunTest(request); + RETURN_IF_ERROR(response.status()); - response.SerializeToString(&serialized_output); + std::string serialized_output; + response->SerializeToString(&serialized_output); - bytes = serialized_output.size(); - CheckedWrite(STDOUT_FILENO, &bytes, sizeof(uint32_t)); - CheckedWrite(STDOUT_FILENO, serialized_output.c_str(), bytes); + uint32_t out_len = static_cast(serialized_output.size()); + RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, &out_len, sizeof(out_len))); + RETURN_IF_ERROR(WriteFd(STDOUT_FILENO, serialized_output.data(), out_len)); - if (verbose) { - fprintf(stderr, "conformance-cpp: request=%s, response=%s\n", - request.ShortDebugString().c_str(), - response.ShortDebugString().c_str()); + if (verbose_) { + GOOGLE_LOG(INFO) << "conformance-cpp: request=" << request.ShortDebugString() + << ", response=" << response->ShortDebugString(); } - - test_count++; - - return true; + return false; } - +} // namespace } // namespace protobuf } // namespace google int main() { - type_resolver = NewTypeResolverForDescriptorPool( - kTypeUrlPrefix, DescriptorPool::generated_pool()); - type_url = new string(GetTypeUrl(TestAllTypesProto3::descriptor())); - while (1) { - if (!google::protobuf::DoTestIo()) { - fprintf(stderr, "conformance-cpp: received EOF from test runner " - "after %d tests, exiting\n", test_count); - return 0; + google::protobuf::Harness harness; + int total_runs = 0; + while (true) { + auto is_done = harness.ServeConformanceRequest(); + if (!is_done.ok()) { + GOOGLE_LOG(FATAL) << is_done.status(); + } + if (*is_done) { + break; } + total_runs++; } + GOOGLE_LOG(INFO) << "conformance-cpp: received EOF from test runner after " + << total_runs << " tests"; } diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index f1e5ac7939..2b8f807951 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -69,15 +69,13 @@ import java.util.Map; import java.util.TreeMap; /** - * All generated protocol message classes extend this class. This class - * implements most of the Message and Builder interfaces using Java reflection. - * Users can ignore this class and pretend that generated messages implement - * the Message interface directly. + * All generated protocol message classes extend this class. This class implements most of the + * Message and Builder interfaces using Java reflection. Users can ignore this class and pretend + * that generated messages implement the Message interface directly. * * @author kenton@google.com Kenton Varda */ -public abstract class GeneratedMessageV3 extends AbstractMessage - implements Serializable { +public abstract class GeneratedMessageV3 extends AbstractMessage implements Serializable { private static final long serialVersionUID = 1L; /** @@ -87,7 +85,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage */ protected static boolean alwaysUseFieldBuilders = false; - /** For use by generated code only. */ + /** For use by generated code only. */ protected UnknownFieldSet unknownFields; protected GeneratedMessageV3() { @@ -100,31 +98,29 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Parser getParserForType() { - throw new UnsupportedOperationException( - "This is supposed to be overridden by subclasses."); + throw new UnsupportedOperationException("This is supposed to be overridden by subclasses."); } - /** - * @see #setAlwaysUseFieldBuildersForTesting(boolean) - */ + /** + * @see #setAlwaysUseFieldBuildersForTesting(boolean) + */ static void enableAlwaysUseFieldBuildersForTesting() { setAlwaysUseFieldBuildersForTesting(true); } /** - * For testing. Allows a test to disable/re-enable the optimization that avoids - * using field builders for nested messages until they are requested. By disabling - * this optimization, existing tests can be reused to test the field builders. - * See {@link RepeatedFieldBuilder} and {@link SingleFieldBuilder}. + * For testing. Allows a test to disable/re-enable the optimization that avoids using field + * builders for nested messages until they are requested. By disabling this optimization, existing + * tests can be reused to test the field builders. See {@link RepeatedFieldBuilder} and {@link + * SingleFieldBuilder}. */ static void setAlwaysUseFieldBuildersForTesting(boolean useBuilders) { alwaysUseFieldBuilders = useBuilders; } /** - * Get the FieldAccessorTable for this type. We can't have the message - * class pass this in to the constructor because of bootstrapping trouble - * with DescriptorProtos. + * Get the FieldAccessorTable for this type. We can't have the message class pass this in to the + * constructor because of bootstrapping trouble with DescriptorProtos. */ protected abstract FieldAccessorTable internalGetFieldAccessorTable(); @@ -149,16 +145,14 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Internal helper to return a modifiable map containing all the fields. - * The returned Map is modifiable so that the caller can add additional - * extension fields to implement {@link #getAllFields()}. + * Internal helper to return a modifiable map containing all the fields. The returned Map is + * modifiable so that the caller can add additional extension fields to implement {@link + * #getAllFields()}. * * @param getBytesForString whether to generate ByteString for string fields */ - private Map getAllFieldsMutable( - boolean getBytesForString) { - final TreeMap result = - new TreeMap(); + private Map getAllFieldsMutable(boolean getBytesForString) { + final TreeMap result = new TreeMap(); final Descriptor descriptor = internalGetFieldAccessorTable().descriptor; final List fields = descriptor.getFields(); @@ -214,8 +208,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // Check that embedded messages are initialized. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isRepeated()) { - @SuppressWarnings("unchecked") final - List messageList = (List) getField(field); + @SuppressWarnings("unchecked") + final List messageList = (List) getField(field); for (final Message element : messageList) { if (!element.isInitialized()) { return false; @@ -234,23 +228,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Map getAllFields() { - return Collections.unmodifiableMap( - getAllFieldsMutable(/* getBytesForString = */ false)); + return Collections.unmodifiableMap(getAllFieldsMutable(/* getBytesForString = */ false)); } /** - * Returns a collection of all the fields in this message which are set - * and their corresponding values. A singular ("required" or "optional") - * field is set iff hasField() returns true for that field. A "repeated" - * field is set iff getRepeatedFieldCount() is greater than zero. The - * values are exactly what would be returned by calling - * {@link #getFieldRaw(Descriptors.FieldDescriptor)} for each field. The map - * is guaranteed to be a sorted map, so iterating over it will return fields - * in order by field number. + * Returns a collection of all the fields in this message which are set and their corresponding + * values. A singular ("required" or "optional") field is set iff hasField() returns true for that + * field. A "repeated" field is set iff getRepeatedFieldCount() is greater than zero. The values + * are exactly what would be returned by calling {@link #getFieldRaw(Descriptors.FieldDescriptor)} + * for each field. The map is guaranteed to be a sorted map, so iterating over it will return + * fields in order by field number. */ Map getAllFieldsRaw() { - return Collections.unmodifiableMap( - getAllFieldsMutable(/* getBytesForString = */ true)); + return Collections.unmodifiableMap(getAllFieldsMutable(/* getBytesForString = */ true)); } @Override @@ -274,12 +264,11 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Obtains the value of the given field, or the default value if it is - * not set. For primitive fields, the boxed primitive value is returned. - * For enum fields, the EnumValueDescriptor for the value is returned. For - * embedded message fields, the sub-message is returned. For repeated - * fields, a java.util.List is returned. For present string fields, a - * ByteString is returned representing the bytes that the field contains. + * Obtains the value of the given field, or the default value if it is not set. For primitive + * fields, the boxed primitive value is returned. For enum fields, the EnumValueDescriptor for the + * value is returned. For embedded message fields, the sub-message is returned. For repeated + * fields, a java.util.List is returned. For present string fields, a ByteString is returned + * representing the bytes that the field contains. */ Object getFieldRaw(final FieldDescriptor field) { return internalGetFieldAccessorTable().getField(field).getRaw(this); @@ -287,20 +276,17 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public int getRepeatedFieldCount(final FieldDescriptor field) { - return internalGetFieldAccessorTable().getField(field) - .getRepeatedCount(this); + return internalGetFieldAccessorTable().getField(field).getRepeatedCount(this); } @Override public Object getRepeatedField(final FieldDescriptor field, final int index) { - return internalGetFieldAccessorTable().getField(field) - .getRepeated(this, index); + return internalGetFieldAccessorTable().getField(field).getRepeated(this, index); } @Override public UnknownFieldSet getUnknownFields() { - throw new UnsupportedOperationException( - "This is supposed to be overridden by subclasses."); + throw new UnsupportedOperationException("This is supposed to be overridden by subclasses."); } /** @@ -342,8 +328,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, InputStream input, - ExtensionRegistryLite extensions) throws IOException { + protected static M parseWithIOException( + Parser parser, InputStream input, ExtensionRegistryLite extensions) throws IOException { try { return parser.parseFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -351,8 +337,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, - CodedInputStream input) throws IOException { + protected static M parseWithIOException( + Parser parser, CodedInputStream input) throws IOException { try { return parser.parseFrom(input); } catch (InvalidProtocolBufferException e) { @@ -360,8 +346,9 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseWithIOException(Parser parser, - CodedInputStream input, ExtensionRegistryLite extensions) throws IOException { + protected static M parseWithIOException( + Parser parser, CodedInputStream input, ExtensionRegistryLite extensions) + throws IOException { try { return parser.parseFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -369,8 +356,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseDelimitedWithIOException(Parser parser, - InputStream input) throws IOException { + protected static M parseDelimitedWithIOException( + Parser parser, InputStream input) throws IOException { try { return parser.parseDelimitedFrom(input); } catch (InvalidProtocolBufferException e) { @@ -378,8 +365,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static M parseDelimitedWithIOException(Parser parser, - InputStream input, ExtensionRegistryLite extensions) throws IOException { + protected static M parseDelimitedWithIOException( + Parser parser, InputStream input, ExtensionRegistryLite extensions) throws IOException { try { return parser.parseDelimitedFrom(input, extensions); } catch (InvalidProtocolBufferException e) { @@ -480,63 +467,53 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } - /** * This class is used to make a generated protected method inaccessible from user's code (e.g., * the {@link #newInstance} method below). When this class is used as a parameter's type in a - * generated protected method, the method is visible to user's code in the same package, but - * since the constructor of this class is private to protobuf runtime, user's code can't obtain - * an instance of this class and as such can't actually make a method call on the protected - * method. + * generated protected method, the method is visible to user's code in the same package, but since + * the constructor of this class is private to protobuf runtime, user's code can't obtain an + * instance of this class and as such can't actually make a method call on the protected method. */ protected static final class UnusedPrivateParameter { static final UnusedPrivateParameter INSTANCE = new UnusedPrivateParameter(); - private UnusedPrivateParameter() { - } + private UnusedPrivateParameter() {} } - /** - * Creates a new instance of this message type. Overridden in the generated code. - */ + /** Creates a new instance of this message type. Overridden in the generated code. */ @SuppressWarnings({"unused"}) protected Object newInstance(UnusedPrivateParameter unused) { throw new UnsupportedOperationException("This method must be overridden by the subclass."); } - /** - * Used by parsing constructors in generated classes. - */ + /** Used by parsing constructors in generated classes. */ protected void makeExtensionsImmutable() { // Noop for messages without extensions. } /** - * TODO(xiaofeng): remove this after b/29368482 is fixed. We need to move this - * interface to AbstractMessage in order to versioning GeneratedMessageV3 but - * this move breaks binary compatibility for AppEngine. After AppEngine is - * fixed we can exclude this from google3. + * TODO(xiaofeng): remove this after b/29368482 is fixed. We need to move this interface to + * AbstractMessage in order to versioning GeneratedMessageV3 but this move breaks binary + * compatibility for AppEngine. After AppEngine is fixed we can exclude this from google3. */ protected interface BuilderParent extends AbstractMessage.BuilderParent {} - /** - * TODO(xiaofeng): remove this together with GeneratedMessageV3.BuilderParent. - */ + /** TODO(xiaofeng): remove this together with GeneratedMessageV3.BuilderParent. */ protected abstract Message.Builder newBuilderForType(BuilderParent parent); @Override protected Message.Builder newBuilderForType(final AbstractMessage.BuilderParent parent) { - return newBuilderForType(new BuilderParent() { - @Override - public void markDirty() { - parent.markDirty(); - } - }); + return newBuilderForType( + new BuilderParent() { + @Override + public void markDirty() { + parent.markDirty(); + } + }); } - @SuppressWarnings("unchecked") - public abstract static class Builder > + public abstract static class Builder> extends AbstractMessage.Builder { private BuilderParent builderParent; @@ -547,8 +524,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // to dispatch dirty invalidations. See GeneratedMessageV3.BuilderListener. private boolean isClean; - private UnknownFieldSet unknownFields = - UnknownFieldSet.getDefaultInstance(); + private UnknownFieldSet unknownFields = UnknownFieldSet.getDefaultInstance(); protected Builder() { this(null); @@ -563,9 +539,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage builderParent = null; } - /** - * Called by the subclass when a message is built. - */ + /** Called by the subclass when a message is built. */ protected void onBuilt() { if (builderParent != null) { markClean(); @@ -573,8 +547,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called by the subclass or a builder to notify us that a message was - * built and may be cached and therefore invalidations are needed. + * Called by the subclass or a builder to notify us that a message was built and may be cached + * and therefore invalidations are needed. */ @Override protected void markClean() { @@ -592,15 +566,14 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public BuilderType clone() { - BuilderType builder = - (BuilderType) getDefaultInstanceForType().newBuilderForType(); + BuilderType builder = (BuilderType) getDefaultInstanceForType().newBuilderForType(); builder.mergeFrom(buildPartial()); return builder; } /** - * Called by the initialization and clear code paths to allow subclasses to - * reset any of their builtin fields back to the initial values. + * Called by the initialization and clear code paths to allow subclasses to reset any of their + * builtin fields back to the initial values. */ @Override public BuilderType clear() { @@ -610,9 +583,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Get the FieldAccessorTable for this type. We can't have the message - * class pass this in to the constructor because of bootstrapping trouble - * with DescriptorProtos. + * Get the FieldAccessorTable for this type. We can't have the message class pass this in to the + * constructor because of bootstrapping trouble with DescriptorProtos. */ protected abstract FieldAccessorTable internalGetFieldAccessorTable(); @@ -628,8 +600,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Internal helper which returns a mutable map. */ private Map getAllFieldsMutable() { - final TreeMap result = - new TreeMap(); + final TreeMap result = new TreeMap(); final Descriptor descriptor = internalGetFieldAccessorTable().descriptor; final List fields = descriptor.getFields(); @@ -681,8 +652,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public Message.Builder getRepeatedFieldBuilder(final FieldDescriptor field, int index) { - return internalGetFieldAccessorTable().getField(field).getRepeatedBuilder( - this, index); + return internalGetFieldAccessorTable().getField(field).getRepeatedBuilder(this, index); } @Override @@ -732,21 +702,18 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public int getRepeatedFieldCount(final FieldDescriptor field) { - return internalGetFieldAccessorTable().getField(field) - .getRepeatedCount(this); + return internalGetFieldAccessorTable().getField(field).getRepeatedCount(this); } @Override public Object getRepeatedField(final FieldDescriptor field, final int index) { - return internalGetFieldAccessorTable().getField(field) - .getRepeated(this, index); + return internalGetFieldAccessorTable().getField(field).getRepeated(this, index); } @Override public BuilderType setRepeatedField( final FieldDescriptor field, final int index, final Object value) { - internalGetFieldAccessorTable().getField(field) - .setRepeated(this, index, value); + internalGetFieldAccessorTable().getField(field).setRepeated(this, index, value); return (BuilderType) this; } @@ -768,20 +735,16 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * This method is obsolete, but we must retain it for compatibility with - * older generated code. + * This method is obsolete, but we must retain it for compatibility with older generated code. */ protected BuilderType setUnknownFieldsProto3(final UnknownFieldSet unknownFields) { return setUnknownFieldsInternal(unknownFields); } @Override - public BuilderType mergeUnknownFields( - final UnknownFieldSet unknownFields) { + public BuilderType mergeUnknownFields(final UnknownFieldSet unknownFields) { return setUnknownFields( - UnknownFieldSet.newBuilder(this.unknownFields) - .mergeFrom(unknownFields) - .build()); + UnknownFieldSet.newBuilder(this.unknownFields).mergeFrom(unknownFields).build()); } @@ -797,16 +760,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // Check that embedded messages are initialized. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isRepeated()) { - @SuppressWarnings("unchecked") final - List messageList = (List) getField(field); + @SuppressWarnings("unchecked") + final List messageList = (List) getField(field); for (final Message element : messageList) { if (!element.isInitialized()) { return false; } } } else { - if (hasField(field) && - !((Message) getField(field)).isInitialized()) { + if (hasField(field) && !((Message) getField(field)).isInitialized()) { return false; } } @@ -821,9 +783,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Implementation of {@link BuilderParent} for giving to our children. This - * small inner class makes it so we don't publicly expose the BuilderParent - * methods. + * Implementation of {@link BuilderParent} for giving to our children. This small inner class + * makes it so we don't publicly expose the BuilderParent methods. */ private class BuilderParentImpl implements BuilderParent { @@ -835,6 +796,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Gets the {@link BuilderParent} for giving to our children. + * * @return The builder parent for our children. */ protected BuilderParent getParentForChildren() { @@ -845,8 +807,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called when a the builder or one of its nested children has changed - * and any parent should be notified of its invalidation. + * Called when a the builder or one of its nested children has changed and any parent should be + * notified of its invalidation. */ protected final void onChanged() { if (isClean && builderParent != null) { @@ -858,22 +820,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Gets the map field with the given field number. This method should be - * overridden in the generated message class if the message contains map - * fields. + * Gets the map field with the given field number. This method should be overridden in the + * generated message class if the message contains map fields. * - * Unlike other field types, reflection support for map fields can't be - * implemented based on generated public API because we need to access a - * map field as a list in reflection API but the generated API only allows - * us to access it as a map. This method returns the underlying map field - * directly and thus enables us to access the map field as a list. + *

Unlike other field types, reflection support for map fields can't be implemented based on + * generated public API because we need to access a map field as a list in reflection API but + * the generated API only allows us to access it as a map. This method returns the underlying + * map field directly and thus enables us to access the map field as a list. */ @SuppressWarnings({"unused", "rawtypes"}) protected MapField internalGetMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } /** Like {@link #internalGetMapField} but return a mutable version. */ @@ -881,36 +840,30 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected MapField internalGetMutableMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } } // ================================================================= // Extensions-related stuff - public interface ExtendableMessageOrBuilder< - MessageType extends ExtendableMessage> extends MessageOrBuilder { + public interface ExtendableMessageOrBuilder + extends MessageOrBuilder { // Re-define for return type covariance. @Override Message getDefaultInstanceForType(); /** Check if a singular extension is present. */ - boolean hasExtension( - ExtensionLite extension); + boolean hasExtension(ExtensionLite extension); /** Get the number of elements in a repeated extension. */ - int getExtensionCount( - ExtensionLite> extension); + int getExtensionCount(ExtensionLite> extension); /** Get the value of an extension. */ - Type getExtension( - ExtensionLite extension); + Type getExtension(ExtensionLite extension); /** Get one element of a repeated extension. */ - Type getExtension( - ExtensionLite> extension, - int index); + Type getExtension(ExtensionLite> extension, int index); /** Check if a singular extension is present. */ boolean hasExtension( @@ -941,16 +894,13 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Generated message classes for message types that contain extension ranges - * subclass this. + * Generated message classes for message types that contain extension ranges subclass this. * - *

This class implements type-safe accessors for extensions. They - * implement all the same operations that you can do with normal fields -- - * e.g. "has", "get", and "getCount" -- but for extensions. The extensions - * are identified using instances of the class {@link GeneratedExtension}; - * the protocol compiler generates a static instance of this class for every - * extension in its input. Through the magic of generics, all is made - * type-safe. + *

This class implements type-safe accessors for extensions. They implement all the same + * operations that you can do with normal fields -- e.g. "has", "get", and "getCount" -- but for + * extensions. The extensions are identified using instances of the class {@link + * GeneratedExtension}; the protocol compiler generates a static instance of this class for every + * extension in its input. Through the magic of generics, all is made type-safe. * *

For example, imagine you have the {@code .proto} file: * @@ -975,10 +925,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage * *

See also {@link ExtendableBuilder}. */ - public abstract static class ExtendableMessage< - MessageType extends ExtendableMessage> - extends GeneratedMessageV3 - implements ExtendableMessageOrBuilder { + public abstract static class ExtendableMessage + extends GeneratedMessageV3 implements ExtendableMessageOrBuilder { private static final long serialVersionUID = 1L; @@ -988,22 +936,20 @@ public abstract class GeneratedMessageV3 extends AbstractMessage this.extensions = FieldSet.newFieldSet(); } - protected ExtendableMessage( - ExtendableBuilder builder) { + protected ExtendableMessage(ExtendableBuilder builder) { super(builder); this.extensions = builder.buildExtensions(); } - private void verifyExtensionContainingType( - final Extension extension) { - if (extension.getDescriptor().getContainingType() != - getDescriptorForType()) { + private void verifyExtensionContainingType(final Extension extension) { + if (extension.getDescriptor().getContainingType() != getDescriptorForType()) { // This can only happen if someone uses unchecked operations. throw new IllegalArgumentException( - "Extension is for type \"" + - extension.getDescriptor().getContainingType().getFullName() + - "\" which does not match message type \"" + - getDescriptorForType().getFullName() + "\"."); + "Extension is for type \"" + + extension.getDescriptor().getContainingType().getFullName() + + "\" which does not match message type \"" + + getDescriptorForType().getFullName() + + "\"."); } } @@ -1041,12 +987,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (value == null) { if (descriptor.isRepeated()) { return (Type) Collections.emptyList(); - } else if (descriptor.getJavaType() == - FieldDescriptor.JavaType.MESSAGE) { + } else if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { return (Type) extension.getMessageDefaultInstance(); } else { - return (Type) extension.fromReflectionType( - descriptor.getDefaultValue()); + return (Type) extension.fromReflectionType(descriptor.getDefaultValue()); } } else { return (Type) extension.fromReflectionType(value); @@ -1062,8 +1006,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage verifyExtensionContainingType(extension); FieldDescriptor descriptor = extension.getDescriptor(); - return (Type) extension.singularFromReflectionType( - extensions.getRepeatedField(descriptor, index)); + return (Type) + extension.singularFromReflectionType(extensions.getRepeatedField(descriptor, index)); } /** Check if a singular extension is present. */ @@ -1128,10 +1072,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedInputStream input, UnknownFieldSet.Builder unknownFields, ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + int tag) + throws IOException { return MessageReflection.mergeFieldFrom( - input, input.shouldDiscardUnknownFields() ? null : unknownFields, extensionRegistry, - getDescriptorForType(), new MessageReflection.ExtensionAdapter(extensions), tag); + input, + input.shouldDiscardUnknownFields() ? null : unknownFields, + extensionRegistry, + getDescriptorForType(), + new MessageReflection.ExtensionAdapter(extensions), + tag); } /** @@ -1143,31 +1092,28 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedInputStream input, UnknownFieldSet.Builder unknownFields, ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + int tag) + throws IOException { return parseUnknownField(input, unknownFields, extensionRegistry, tag); } - /** - * Used by parsing constructors in generated classes. - */ + /** Used by parsing constructors in generated classes. */ @Override protected void makeExtensionsImmutable() { extensions.makeImmutable(); } /** - * Used by subclasses to serialize extensions. Extension ranges may be - * interleaved with field numbers, but we must write them in canonical - * (sorted by field number) order. ExtensionWriter helps us write - * individual ranges of extensions at once. + * Used by subclasses to serialize extensions. Extension ranges may be interleaved with field + * numbers, but we must write them in canonical (sorted by field number) order. ExtensionWriter + * helps us write individual ranges of extensions at once. */ protected class ExtensionWriter { // Imagine how much simpler this code would be if Java iterators had // a way to get the next element without advancing the iterator. - private final Iterator> iter = - extensions.iterator(); + private final Iterator> iter = extensions.iterator(); private Map.Entry next; private final boolean messageSetWireFormat; @@ -1178,19 +1124,18 @@ public abstract class GeneratedMessageV3 extends AbstractMessage this.messageSetWireFormat = messageSetWireFormat; } - public void writeUntil(final int end, final CodedOutputStream output) - throws IOException { + public void writeUntil(final int end, final CodedOutputStream output) throws IOException { while (next != null && next.getKey().getNumber() < end) { FieldDescriptor descriptor = next.getKey(); - if (messageSetWireFormat && descriptor.getLiteJavaType() == - WireFormat.JavaType.MESSAGE && - !descriptor.isRepeated()) { + if (messageSetWireFormat + && descriptor.getLiteJavaType() == WireFormat.JavaType.MESSAGE + && !descriptor.isRepeated()) { if (next instanceof LazyField.LazyEntry) { - output.writeRawMessageSetExtension(descriptor.getNumber(), + output.writeRawMessageSetExtension( + descriptor.getNumber(), ((LazyField.LazyEntry) next).getField().toByteString()); } else { - output.writeMessageSetExtension(descriptor.getNumber(), - (Message) next.getValue()); + output.writeMessageSetExtension(descriptor.getNumber(), (Message) next.getValue()); } } else { // TODO(xiangl): Taken care of following code, it may cause @@ -1214,6 +1159,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected ExtensionWriter newExtensionWriter() { return new ExtensionWriter(false); } + protected ExtensionWriter newMessageSetExtensionWriter() { return new ExtensionWriter(true); } @@ -1222,6 +1168,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected int extensionsSerializedSize() { return extensions.getSerializedSize(); } + protected int extensionsSerializedSizeAsMessageSet() { return extensions.getMessageSetSerializedSize(); } @@ -1293,8 +1240,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeatedField(final FieldDescriptor field, - final int index) { + public Object getRepeatedField(final FieldDescriptor field, final int index) { if (field.isExtension()) { verifyContainingType(field); return extensions.getRepeatedField(field, index); @@ -1305,23 +1251,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private void verifyContainingType(final FieldDescriptor field) { if (field.getContainingType() != getDescriptorForType()) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } } } /** - * Generated message builders for message types that contain extension ranges - * subclass this. + * Generated message builders for message types that contain extension ranges subclass this. * - *

This class implements type-safe accessors for extensions. They - * implement all the same operations that you can do with normal fields -- - * e.g. "get", "set", and "add" -- but for extensions. The extensions are - * identified using instances of the class {@link GeneratedExtension}; the - * protocol compiler generates a static instance of this class for every - * extension in its input. Through the magic of generics, all is made - * type-safe. + *

This class implements type-safe accessors for extensions. They implement all the same + * operations that you can do with normal fields -- e.g. "get", "set", and "add" -- but for + * extensions. The extensions are identified using instances of the class {@link + * GeneratedExtension}; the protocol compiler generates a static instance of this class for every + * extension in its input. Through the magic of generics, all is made type-safe. * *

For example, imagine you have the {@code .proto} file: * @@ -1350,17 +1292,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage */ @SuppressWarnings("unchecked") public abstract static class ExtendableBuilder< - MessageType extends ExtendableMessage, - BuilderType extends ExtendableBuilder> - extends Builder - implements ExtendableMessageOrBuilder { + MessageType extends ExtendableMessage, + BuilderType extends ExtendableBuilder> + extends Builder implements ExtendableMessageOrBuilder { private FieldSet.Builder extensions; protected ExtendableBuilder() {} - protected ExtendableBuilder( - BuilderParent parent) { + protected ExtendableBuilder(BuilderParent parent) { super(parent); } @@ -1381,16 +1321,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - private void verifyExtensionContainingType( - final Extension extension) { - if (extension.getDescriptor().getContainingType() != - getDescriptorForType()) { + private void verifyExtensionContainingType(final Extension extension) { + if (extension.getDescriptor().getContainingType() != getDescriptorForType()) { // This can only happen if someone uses unchecked operations. throw new IllegalArgumentException( - "Extension is for type \"" + - extension.getDescriptor().getContainingType().getFullName() + - "\" which does not match message type \"" + - getDescriptorForType().getFullName() + "\"."); + "Extension is for type \"" + + extension.getDescriptor().getContainingType().getFullName() + + "\" which does not match message type \"" + + getDescriptorForType().getFullName() + + "\"."); } } @@ -1425,12 +1364,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (value == null) { if (descriptor.isRepeated()) { return (Type) Collections.emptyList(); - } else if (descriptor.getJavaType() == - FieldDescriptor.JavaType.MESSAGE) { + } else if (descriptor.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { return (Type) extension.getMessageDefaultInstance(); } else { - return (Type) extension.fromReflectionType( - descriptor.getDefaultValue()); + return (Type) extension.fromReflectionType(descriptor.getDefaultValue()); } } else { return (Type) extension.fromReflectionType(value); @@ -1454,8 +1391,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Set the value of an extension. */ public final BuilderType setExtension( - final ExtensionLite extensionLite, - final Type value) { + final ExtensionLite extensionLite, final Type value) { Extension extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); @@ -1469,30 +1405,27 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Set the value of one element of a repeated extension. */ public final BuilderType setExtension( final ExtensionLite> extensionLite, - final int index, final Type value) { + final int index, + final Type value) { Extension> extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); ensureExtensionsIsMutable(); final FieldDescriptor descriptor = extension.getDescriptor(); - extensions.setRepeatedField( - descriptor, index, - extension.singularToReflectionType(value)); + extensions.setRepeatedField(descriptor, index, extension.singularToReflectionType(value)); onChanged(); return (BuilderType) this; } /** Append a value to a repeated extension. */ public final BuilderType addExtension( - final ExtensionLite> extensionLite, - final Type value) { + final ExtensionLite> extensionLite, final Type value) { Extension> extension = checkNotLite(extensionLite); verifyExtensionContainingType(extension); ensureExtensionsIsMutable(); final FieldDescriptor descriptor = extension.getDescriptor(); - extensions.addRepeatedField( - descriptor, extension.singularToReflectionType(value)); + extensions.addRepeatedField(descriptor, extension.singularToReflectionType(value)); onChanged(); return (BuilderType) this; } @@ -1603,8 +1536,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called by the build code path to create a copy of the extensions for - * building the message. + * Called by the build code path to create a copy of the extensions for building the message. */ private FieldSet buildExtensions() { return extensions == null @@ -1694,8 +1626,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeatedField(final FieldDescriptor field, - final int index) { + public Object getRepeatedField(final FieldDescriptor field, final int index) { if (field.isExtension()) { verifyContainingType(field); if (extensions == null) { @@ -1744,8 +1675,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType setField(final FieldDescriptor field, - final Object value) { + public BuilderType setField(final FieldDescriptor field, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1771,8 +1701,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType setRepeatedField(final FieldDescriptor field, - final int index, final Object value) { + public BuilderType setRepeatedField( + final FieldDescriptor field, final int index, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1785,8 +1715,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public BuilderType addRepeatedField(final FieldDescriptor field, - final Object value) { + public BuilderType addRepeatedField(final FieldDescriptor field, final Object value) { if (field.isExtension()) { verifyContainingType(field); ensureExtensionsIsMutable(); @@ -1817,8 +1746,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private void verifyContainingType(final FieldDescriptor field) { if (field.getContainingType() != getDescriptorForType()) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } } } @@ -1826,8 +1754,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // ----------------------------------------------------------------- /** - * Gets the descriptor for an extension. The implementation depends on whether - * the extension is scoped in the top level of a file or scoped in a Message. + * Gets the descriptor for an extension. The implementation depends on whether the extension is + * scoped in the top level of a file or scoped in a Message. */ static interface ExtensionDescriptorRetriever { FieldDescriptor getDescriptor(); @@ -1844,8 +1772,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return clazz.getMethod(name, params); } catch (NoSuchMethodException e) { throw new RuntimeException( - "Generated message class \"" + clazz.getName() + - "\" missing method \"" + name + "\".", e); + "Generated message class \"" + clazz.getName() + "\" missing method \"" + name + "\".", + e); } } @@ -1857,8 +1785,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return method.invoke(object, params); } catch (IllegalAccessException e) { throw new RuntimeException( - "Couldn't use Java reflection to implement protocol message " + - "reflection.", e); + "Couldn't use Java reflection to implement protocol message " + "reflection.", e); } catch (InvocationTargetException e) { final Throwable cause = e.getCause(); if (cause instanceof RuntimeException) { @@ -1867,7 +1794,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage throw (Error) cause; } else { throw new RuntimeException( - "Unexpected exception thrown by generated accessor method.", cause); + "Unexpected exception thrown by generated accessor method.", cause); } } } @@ -1885,25 +1812,24 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected MapField internalGetMapField(int fieldNumber) { // Note that we can't use descriptor names here because this method will // be called when descriptor is being initialized. - throw new RuntimeException( - "No map fields found in " + getClass().getName()); + throw new RuntimeException("No map fields found in " + getClass().getName()); } /** - * Users should ignore this class. This class provides the implementation - * with access to the fields of a message object using Java reflection. + * Users should ignore this class. This class provides the implementation with access to the + * fields of a message object using Java reflection. */ public static final class FieldAccessorTable { /** - * Construct a FieldAccessorTable for a particular message class. Only - * one FieldAccessorTable should ever be constructed per class. + * Construct a FieldAccessorTable for a particular message class. Only one FieldAccessorTable + * should ever be constructed per class. * - * @param descriptor The type's descriptor. - * @param camelCaseNames The camelcase names of all fields in the message. - * These are used to derive the accessor method names. - * @param messageClass The message type. - * @param builderClass The builder type. + * @param descriptor The type's descriptor. + * @param camelCaseNames The camelcase names of all fields in the message. These are used to + * derive the accessor method names. + * @param messageClass The message type. + * @param builderClass The builder type. */ public FieldAccessorTable( final Descriptor descriptor, @@ -1915,12 +1841,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Construct a FieldAccessorTable for a particular message class without - * initializing FieldAccessors. + * Construct a FieldAccessorTable for a particular message class without initializing + * FieldAccessors. */ - public FieldAccessorTable( - final Descriptor descriptor, - final String[] camelCaseNames) { + public FieldAccessorTable(final Descriptor descriptor, final String[] camelCaseNames) { this.descriptor = descriptor; this.camelCaseNames = camelCaseNames; fields = new FieldAccessor[descriptor.getFields().size()]; @@ -1931,16 +1855,19 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Ensures the field accessors are initialized. This method is thread-safe. * - * @param messageClass The message type. - * @param builderClass The builder type. + * @param messageClass The message type. + * @param builderClass The builder type. * @return this */ public FieldAccessorTable ensureFieldAccessorsInitialized( - Class messageClass, - Class builderClass) { - if (initialized) { return this; } + Class messageClass, Class builderClass) { + if (initialized) { + return this; + } synchronized (this) { - if (initialized) { return this; } + if (initialized) { + return this; + } int fieldsSize = fields.length; for (int i = 0; i < fieldsSize; i++) { FieldDescriptor field = descriptor.getFields().get(i); @@ -1952,36 +1879,54 @@ public abstract class GeneratedMessageV3 extends AbstractMessage if (field.isRepeated()) { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { if (field.isMapField()) { - fields[i] = new MapFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new MapFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = new RepeatedMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedMessageFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); } } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new RepeatedEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedEnumFieldAccessor( + field, camelCaseNames[i], messageClass, builderClass); } else { - fields[i] = new RepeatedFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass); + fields[i] = + new RepeatedFieldAccessor(field, camelCaseNames[i], messageClass, builderClass); } } else { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - fields[i] = new SingularMessageFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularMessageFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else if (field.getJavaType() == FieldDescriptor.JavaType.ENUM) { - fields[i] = new SingularEnumFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularEnumFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else if (field.getJavaType() == FieldDescriptor.JavaType.STRING) { - fields[i] = new SingularStringFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularStringFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } else { - fields[i] = new SingularFieldAccessor( - field, camelCaseNames[i], messageClass, builderClass, - containingOneofCamelCaseName); + fields[i] = + new SingularFieldAccessor( + field, + camelCaseNames[i], + messageClass, + builderClass, + containingOneofCamelCaseName); } } } @@ -2007,13 +1952,11 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Get the FieldAccessor for a particular field. */ private FieldAccessor getField(final FieldDescriptor field) { if (field.getContainingType() != descriptor) { - throw new IllegalArgumentException( - "FieldDescriptor does not match message type."); + throw new IllegalArgumentException("FieldDescriptor does not match message type."); } else if (field.isExtension()) { // If this type had extensions, it would subclass ExtendableMessage, // which overrides the reflection interface to handle extensions. - throw new IllegalArgumentException( - "This type does not have extensions."); + throw new IllegalArgumentException("This type does not have extensions."); } return fields[field.getIndex()]; } @@ -2021,38 +1964,53 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** Get the OneofAccessor for a particular oneof. */ private OneofAccessor getOneof(final OneofDescriptor oneof) { if (oneof.getContainingType() != descriptor) { - throw new IllegalArgumentException( - "OneofDescriptor does not match message type."); + throw new IllegalArgumentException("OneofDescriptor does not match message type."); } return oneofs[oneof.getIndex()]; } /** - * Abstract interface that provides access to a single field. This is - * implemented differently depending on the field type and cardinality. + * Abstract interface that provides access to a single field. This is implemented differently + * depending on the field type and cardinality. */ private interface FieldAccessor { Object get(GeneratedMessageV3 message); + Object get(GeneratedMessageV3.Builder builder); + Object getRaw(GeneratedMessageV3 message); + Object getRaw(GeneratedMessageV3.Builder builder); + void set(Builder builder, Object value); + Object getRepeated(GeneratedMessageV3 message, int index); + Object getRepeated(GeneratedMessageV3.Builder builder, int index); + Object getRepeatedRaw(GeneratedMessageV3 message, int index); + Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index); - void setRepeated(Builder builder, - int index, Object value); + + void setRepeated(Builder builder, int index, Object value); + void addRepeated(Builder builder, Object value); + boolean has(GeneratedMessageV3 message); + boolean has(GeneratedMessageV3.Builder builder); + int getRepeatedCount(GeneratedMessageV3 message); + int getRepeatedCount(GeneratedMessageV3.Builder builder); + void clear(Builder builder); + Message.Builder newBuilder(); + Message.Builder getBuilder(GeneratedMessageV3.Builder builder); - Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, - int index); + + Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index); } /** OneofAccessor provides access to a single oneof. */ @@ -2280,48 +2238,59 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object get(final GeneratedMessageV3 message) { return invoker.get(message); } + @Override public Object get(GeneratedMessageV3.Builder builder) { return invoker.get(builder); } + @Override public Object getRaw(final GeneratedMessageV3 message) { return get(message); } + @Override public Object getRaw(GeneratedMessageV3.Builder builder) { return get(builder); } + @Override public void set(final Builder builder, final Object value) { invoker.set(builder, value); } + @Override public Object getRepeated(final GeneratedMessageV3 message, final int index) { throw new UnsupportedOperationException("getRepeatedField() called on a singular field."); } + @Override public Object getRepeatedRaw(final GeneratedMessageV3 message, final int index) { throw new UnsupportedOperationException( "getRepeatedFieldRaw() called on a singular field."); } + @Override public Object getRepeated(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException("getRepeatedField() called on a singular field."); } + @Override public Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( "getRepeatedFieldRaw() called on a singular field."); } + @Override public void setRepeated(final Builder builder, final int index, final Object value) { throw new UnsupportedOperationException("setRepeatedField() called on a singular field."); } + @Override public void addRepeated(final Builder builder, final Object value) { throw new UnsupportedOperationException("addRepeatedField() called on a singular field."); } + @Override public boolean has(final GeneratedMessageV3 message) { if (!hasHasMethod) { @@ -2332,6 +2301,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } return invoker.has(message); } + @Override public boolean has(GeneratedMessageV3.Builder builder) { if (!hasHasMethod) { @@ -2342,29 +2312,35 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } return invoker.has(builder); } + @Override public int getRepeatedCount(final GeneratedMessageV3 message) { throw new UnsupportedOperationException( "getRepeatedFieldSize() called on a singular field."); } + @Override public int getRepeatedCount(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException( "getRepeatedFieldSize() called on a singular field."); } + @Override public void clear(final Builder builder) { invoker.clear(builder); } + @Override public Message.Builder newBuilder() { throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("getFieldBuilder() called on a non-Message type."); } + @Override public Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( @@ -2435,8 +2411,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public Object getRepeated( - final GeneratedMessageV3 message, final int index) { + public Object getRepeated(final GeneratedMessageV3 message, final int index) { return invokeOrDie(getRepeatedMethod, message, index); } @@ -2453,8 +2428,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } @Override - public void addRepeated( - final GeneratedMessageV3.Builder builder, final Object value) { + public void addRepeated(final GeneratedMessageV3.Builder builder, final Object value) { // TODO(b/230609037): remove the unused variable Object unused = invokeOrDie(addRepeatedMethod, builder, value); } @@ -2480,7 +2454,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage protected final MethodInvoker invoker; RepeatedFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { ReflectionInvoker reflectionInvoker = @@ -2497,18 +2472,22 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object get(final GeneratedMessageV3 message) { return invoker.get(message); } + @Override public Object get(GeneratedMessageV3.Builder builder) { return invoker.get(builder); } + @Override public Object getRaw(final GeneratedMessageV3 message) { return get(message); } + @Override public Object getRaw(GeneratedMessageV3.Builder builder) { return get(builder); } + @Override public void set(final Builder builder, final Object value) { // Add all the elements individually. This serves two purposes: @@ -2520,59 +2499,73 @@ public abstract class GeneratedMessageV3 extends AbstractMessage addRepeated(builder, element); } } + @Override public Object getRepeated(final GeneratedMessageV3 message, final int index) { return invoker.getRepeated(message, index); } + @Override public Object getRepeated(GeneratedMessageV3.Builder builder, int index) { return invoker.getRepeated(builder, index); } + @Override public Object getRepeatedRaw(GeneratedMessageV3 message, int index) { return getRepeated(message, index); } + @Override public Object getRepeatedRaw(GeneratedMessageV3.Builder builder, int index) { return getRepeated(builder, index); } + @Override public void setRepeated(final Builder builder, final int index, final Object value) { invoker.setRepeated(builder, index, value); } + @Override public void addRepeated(final Builder builder, final Object value) { invoker.addRepeated(builder, value); } + @Override public boolean has(final GeneratedMessageV3 message) { throw new UnsupportedOperationException("hasField() called on a repeated field."); } + @Override public boolean has(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("hasField() called on a repeated field."); } + @Override public int getRepeatedCount(final GeneratedMessageV3 message) { return invoker.getRepeatedCount(message); } + @Override public int getRepeatedCount(GeneratedMessageV3.Builder builder) { return invoker.getRepeatedCount(builder); } + @Override public void clear(final Builder builder) { invoker.clear(builder); } + @Override public Message.Builder newBuilder() { throw new UnsupportedOperationException( "newBuilderForField() called on a non-Message type."); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { throw new UnsupportedOperationException("getFieldBuilder() called on a non-Message type."); } + @Override public Message.Builder getRepeatedBuilder(GeneratedMessageV3.Builder builder, int index) { throw new UnsupportedOperationException( @@ -2582,16 +2575,15 @@ public abstract class GeneratedMessageV3 extends AbstractMessage private static class MapFieldAccessor implements FieldAccessor { MapFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { field = descriptor; - Method getDefaultInstanceMethod = - getMethodOrDie(messageClass, "getDefaultInstance"); - MapField defaultMapField = getMapField( - (GeneratedMessageV3) invokeOrDie(getDefaultInstanceMethod, null)); - mapEntryMessageDefaultInstance = - defaultMapField.getMapEntryMessageDefaultInstance(); + Method getDefaultInstanceMethod = getMethodOrDie(messageClass, "getDefaultInstance"); + MapField defaultMapField = + getMapField((GeneratedMessageV3) invokeOrDie(getDefaultInstanceMethod, null)); + mapEntryMessageDefaultInstance = defaultMapField.getMapEntryMessageDefaultInstance(); } private final FieldDescriptor field; @@ -2605,10 +2597,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return (MapField) builder.internalGetMapField(field.getNumber()); } - private MapField getMutableMapField( - GeneratedMessageV3.Builder builder) { - return (MapField) builder.internalGetMutableMapField( - field.getNumber()); + private MapField getMutableMapField(GeneratedMessageV3.Builder builder) { + return (MapField) builder.internalGetMutableMapField(field.getNumber()); } private Message coerceType(Message value) { @@ -2695,14 +2685,12 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public boolean has(GeneratedMessageV3 message) { - throw new UnsupportedOperationException( - "hasField() is not supported for repeated fields."); + throw new UnsupportedOperationException("hasField() is not supported for repeated fields."); } @Override public boolean has(Builder builder) { - throw new UnsupportedOperationException( - "hasField() is not supported for repeated fields."); + throw new UnsupportedOperationException("hasField() is not supported for repeated fields."); } @Override @@ -2727,8 +2715,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage @Override public com.google.protobuf.Message.Builder getBuilder(Builder builder) { - throw new UnsupportedOperationException( - "Nested builder not supported for map fields."); + throw new UnsupportedOperationException("Nested builder not supported for map fields."); } @Override @@ -2739,10 +2726,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // --------------------------------------------------------------- - private static final class SingularEnumFieldAccessor - extends SingularFieldAccessor { + private static final class SingularEnumFieldAccessor extends SingularFieldAccessor { SingularEnumFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { @@ -2755,12 +2742,9 @@ public abstract class GeneratedMessageV3 extends AbstractMessage supportUnknownEnumValue = descriptor.getFile().supportsUnknownEnumValue(); if (supportUnknownEnumValue) { - getValueMethod = - getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); - getValueMethodBuilder = - getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); - setValueMethod = - getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class); + getValueMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Value"); + getValueMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Value"); + setValueMethod = getMethodOrDie(builderClass, "set" + camelCaseName + "Value", int.class); } } @@ -2804,10 +2788,10 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - private static final class RepeatedEnumFieldAccessor - extends RepeatedFieldAccessor { + private static final class RepeatedEnumFieldAccessor extends RepeatedFieldAccessor { RepeatedEnumFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { super(descriptor, camelCaseName, messageClass, builderClass); @@ -2829,6 +2813,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage getMethodOrDie(builderClass, "add" + camelCaseName + "Value", int.class); } } + private EnumDescriptor enumDescriptor; private final Method valueOfMethod; @@ -2894,6 +2879,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } super.setRepeated(builder, index, invokeOrDie(valueOfMethod, null, value)); } + @Override public void addRepeated(final Builder builder, final Object value) { if (supportUnknownEnumValue) { @@ -2912,29 +2898,25 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Field accessor for string fields. * - *

This class makes getFooBytes() and setFooBytes() available for - * reflection API so that reflection based serialize/parse functions can - * access the raw bytes of the field to preserve non-UTF8 bytes in the - * string. + *

This class makes getFooBytes() and setFooBytes() available for reflection API so that + * reflection based serialize/parse functions can access the raw bytes of the field to preserve + * non-UTF8 bytes in the string. * - *

This ensures the serialize/parse round-trip safety, which is important - * for servers which forward messages. + *

This ensures the serialize/parse round-trip safety, which is important for servers which + * forward messages. */ - private static final class SingularStringFieldAccessor - extends SingularFieldAccessor { + private static final class SingularStringFieldAccessor extends SingularFieldAccessor { SingularStringFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { - super(descriptor, camelCaseName, messageClass, builderClass, - containingOneofCamelCaseName); - getBytesMethod = getMethodOrDie(messageClass, - "get" + camelCaseName + "Bytes"); - getBytesMethodBuilder = getMethodOrDie(builderClass, - "get" + camelCaseName + "Bytes"); - setBytesMethodBuilder = getMethodOrDie(builderClass, - "set" + camelCaseName + "Bytes", ByteString.class); + super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); + getBytesMethod = getMethodOrDie(messageClass, "get" + camelCaseName + "Bytes"); + getBytesMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Bytes"); + setBytesMethodBuilder = + getMethodOrDie(builderClass, "set" + camelCaseName + "Bytes", ByteString.class); } private final Method getBytesMethod; @@ -2964,19 +2946,17 @@ public abstract class GeneratedMessageV3 extends AbstractMessage // --------------------------------------------------------------- - private static final class SingularMessageFieldAccessor - extends SingularFieldAccessor { + private static final class SingularMessageFieldAccessor extends SingularFieldAccessor { SingularMessageFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass, final String containingOneofCamelCaseName) { - super(descriptor, camelCaseName, messageClass, builderClass, - containingOneofCamelCaseName); + super(descriptor, camelCaseName, messageClass, builderClass, containingOneofCamelCaseName); newBuilderMethod = getMethodOrDie(type, "newBuilder"); - getBuilderMethodBuilder = - getMethodOrDie(builderClass, "get" + camelCaseName + "Builder"); + getBuilderMethodBuilder = getMethodOrDie(builderClass, "get" + camelCaseName + "Builder"); } private final Method newBuilderMethod; @@ -3000,27 +2980,29 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public void set(final Builder builder, final Object value) { super.set(builder, coerceType(value)); } + @Override public Message.Builder newBuilder() { return (Message.Builder) invokeOrDie(newBuilderMethod, null); } + @Override public Message.Builder getBuilder(GeneratedMessageV3.Builder builder) { return (Message.Builder) invokeOrDie(getBuilderMethodBuilder, builder); } } - private static final class RepeatedMessageFieldAccessor - extends RepeatedFieldAccessor { + private static final class RepeatedMessageFieldAccessor extends RepeatedFieldAccessor { RepeatedMessageFieldAccessor( - final FieldDescriptor descriptor, final String camelCaseName, + final FieldDescriptor descriptor, + final String camelCaseName, final Class messageClass, final Class builderClass) { super(descriptor, camelCaseName, messageClass, builderClass); newBuilderMethod = getMethodOrDie(type, "newBuilder"); - getBuilderMethodBuilder = getMethodOrDie(builderClass, - "get" + camelCaseName + "Builder", Integer.TYPE); + getBuilderMethodBuilder = + getMethodOrDie(builderClass, "get" + camelCaseName + "Builder", Integer.TYPE); } private final Method newBuilderMethod; @@ -3044,27 +3026,30 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public void setRepeated(final Builder builder, final int index, final Object value) { super.setRepeated(builder, index, coerceType(value)); } + @Override public void addRepeated(final Builder builder, final Object value) { super.addRepeated(builder, coerceType(value)); } + @Override public Message.Builder newBuilder() { return (Message.Builder) invokeOrDie(newBuilderMethod, null); } + @Override public Message.Builder getRepeatedBuilder( final GeneratedMessageV3.Builder builder, final int index) { - return (Message.Builder) invokeOrDie( - getBuilderMethodBuilder, builder, index); + return (Message.Builder) invokeOrDie(getBuilderMethodBuilder, builder, index); } } } /** - * Replaces this object in the output stream with a serialized form. - * Part of Java's serialization magic. Generated sub-classes must override - * this method by calling {@code return super.writeReplace();} + * Replaces this object in the output stream with a serialized form. Part of Java's serialization + * magic. Generated sub-classes must override this method by calling {@code return + * super.writeReplace();} + * * @return a SerializedForm of this message */ protected Object writeReplace() throws ObjectStreamException { @@ -3116,8 +3101,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } } - protected static void writeStringNoTag( - CodedOutputStream output, final Object value) throws IOException { + protected static void writeStringNoTag(CodedOutputStream output, final Object value) + throws IOException { if (value instanceof String) { output.writeStringNoTag((String) value); } else { @@ -3129,7 +3114,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage CodedOutputStream out, MapField field, MapEntry defaultEntry, - int fieldNumber) throws IOException { + int fieldNumber) + throws IOException { Map m = field.getMap(); if (!out.isSerializationDeterministic()) { serializeMapTo(out, m, defaultEntry, fieldNumber); @@ -3144,11 +3130,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } Arrays.sort(keys); for (int key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3171,11 +3154,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } Arrays.sort(keys); for (long key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3197,11 +3177,8 @@ public abstract class GeneratedMessageV3 extends AbstractMessage keys = m.keySet().toArray(keys); Arrays.sort(keys); for (String key : keys) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } @@ -3228,28 +3205,23 @@ public abstract class GeneratedMessageV3 extends AbstractMessage boolean key) throws IOException { if (m.containsKey(key)) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() - .setKey(key) - .setValue(m.get(key)) - .build()); + out.writeMessage( + fieldNumber, defaultEntry.newBuilderForType().setKey(key).setValue(m.get(key)).build()); } } /** Serialize the map using the iteration order. */ private static void serializeMapTo( - CodedOutputStream out, - Map m, - MapEntry defaultEntry, - int fieldNumber) + CodedOutputStream out, Map m, MapEntry defaultEntry, int fieldNumber) throws IOException { for (Map.Entry entry : m.entrySet()) { - out.writeMessage(fieldNumber, - defaultEntry.newBuilderForType() + out.writeMessage( + fieldNumber, + defaultEntry + .newBuilderForType() .setKey(entry.getKey()) .setValue(entry.getValue()) .build()); } } } - diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index a88baca61c..7180ce3fc0 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -463,15 +463,15 @@ public class DescriptorsTest { /** Tests that parsing an unknown enum throws an exception */ @Test public void testParseUnknownEnum() { - FieldDescriptorProto.Builder field = FieldDescriptorProto.newBuilder() - .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) - .setTypeName("UnknownEnum") - .setType(FieldDescriptorProto.Type.TYPE_ENUM) - .setName("bar") - .setNumber(1); - DescriptorProto.Builder messageType = DescriptorProto.newBuilder() - .setName("Foo") - .addField(field); + FieldDescriptorProto.Builder field = + FieldDescriptorProto.newBuilder() + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setTypeName("UnknownEnum") + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setName("bar") + .setNumber(1); + DescriptorProto.Builder messageType = + DescriptorProto.newBuilder().setName("Foo").addField(field); FileDescriptorProto fooProto = FileDescriptorProto.newBuilder() .setName("foo.proto") @@ -486,7 +486,6 @@ public class DescriptorsTest { } } - /** * Tests the translate/crosslink for an example where a message field's name and type name are the * same. diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index f4c4baf4a2..01c2f4b39a 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -1943,7 +1943,7 @@ public class GeneratedMessageTest { @Test public void - extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() { + extendableBuilder_extensionFieldContainingBuilder_setRepeatedFieldOverwritesElement() { TestAllExtensions.Builder builder = TestAllExtensions.newBuilder(); builder.addRepeatedField(REPEATED_NESTED_MESSAGE_EXTENSION, NestedMessage.getDefaultInstance()); // Calling getRepeatedFieldBuilder and ignoring the returned Builder should have no diff --git a/src/Makefile.am b/src/Makefile.am index f88b75e6f4..42975d11e2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -429,6 +429,8 @@ libprotoc_la_SOURCES = \ google/protobuf/compiler/java/message_field_lite.h \ google/protobuf/compiler/java/message_lite.cc \ google/protobuf/compiler/java/message_lite.h \ + google/protobuf/compiler/java/message_serialization.cc \ + google/protobuf/compiler/java/message_serialization.h \ google/protobuf/compiler/java/name_resolver.cc \ google/protobuf/compiler/java/name_resolver.h \ google/protobuf/compiler/java/options.h \ diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 620755aa29..c849a6236e 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -369,6 +369,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/primitive_field_lite.cc @@ -461,6 +462,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_field_lite.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_lite.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/name_resolver.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/names.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/options.h diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h index d02ad93985..76727688b5 100644 --- a/src/google/protobuf/arena_impl.h +++ b/src/google/protobuf/arena_impl.h @@ -55,6 +55,13 @@ namespace google { namespace protobuf { namespace internal { +// To prevent sharing cache lines between threads +#ifdef __cpp_aligned_new +enum { kCacheAlignment = 64 }; +#else +enum { kCacheAlignment = alignof(max_align_t) }; // do the best we can +#endif + inline constexpr size_t AlignUpTo8(size_t n) { // Align n to next multiple of 8 (from Hacker's Delight, Chapter 3.) return (n + 7) & static_cast(-8); @@ -497,10 +504,10 @@ class PROTOBUF_EXPORT ThreadSafeArena { // have fallback function calls in tail position. This substantially improves // code for the happy path. PROTOBUF_NDEBUG_INLINE bool MaybeAllocateAligned(size_t n, void** out) { - SerialArena* a; + SerialArena* arena; if (PROTOBUF_PREDICT_TRUE(!alloc_policy_.should_record_allocs() && - GetSerialArenaFromThreadCache(&a))) { - return a->MaybeAllocateAligned(n, out); + GetSerialArenaFromThreadCache(&arena))) { + return arena->MaybeAllocateAligned(n, out); } return false; } @@ -564,7 +571,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { // fast path optimizes the case where a single thread uses multiple arenas. ThreadCache* tc = &thread_cache(); SerialArena* serial = hint_.load(std::memory_order_acquire); - if (PROTOBUF_PREDICT_TRUE(serial != NULL && serial->owner() == tc)) { + if (PROTOBUF_PREDICT_TRUE(serial != nullptr && serial->owner() == tc)) { *arena = serial; return true; } @@ -602,7 +609,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { #ifdef _MSC_VER #pragma warning(disable : 4324) #endif - struct alignas(64) ThreadCache { + struct alignas(kCacheAlignment) ThreadCache { #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) // If we are using the ThreadLocalStorage class to store the ThreadCache, // then the ThreadCache's default constructor has to be responsible for @@ -610,7 +617,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { ThreadCache() : next_lifecycle_id(0), last_lifecycle_id_seen(-1), - last_serial_arena(NULL) {} + last_serial_arena(nullptr) {} #endif // Number of per-thread lifecycle IDs to reserve. Must be power of two. @@ -633,7 +640,7 @@ class PROTOBUF_EXPORT ThreadSafeArena { #ifdef _MSC_VER #pragma warning(disable : 4324) #endif - struct alignas(64) CacheAlignedLifecycleIdGenerator { + struct alignas(kCacheAlignment) CacheAlignedLifecycleIdGenerator { std::atomic id; }; static CacheAlignedLifecycleIdGenerator lifecycle_id_generator_; diff --git a/src/google/protobuf/arenastring.cc b/src/google/protobuf/arenastring.cc index d6bef1b273..65da0d3d4d 100644 --- a/src/google/protobuf/arenastring.cc +++ b/src/google/protobuf/arenastring.cc @@ -200,7 +200,7 @@ std::string* ArenaStringPtr::Release() { if (IsDefault()) return nullptr; std::string* released = tagged_ptr_.Get(); - if (!tagged_ptr_.IsAllocated()) { + if (tagged_ptr_.IsArena()) { released = tagged_ptr_.IsMutable() ? new std::string(std::move(*released)) : new std::string(*released); } @@ -229,9 +229,7 @@ void ArenaStringPtr::SetAllocated(std::string* value, Arena* arena) { } void ArenaStringPtr::Destroy() { - if (tagged_ptr_.IsAllocated()) { - delete tagged_ptr_.Get(); - } + delete tagged_ptr_.GetIfAllocated(); } void ArenaStringPtr::ClearToEmpty() { diff --git a/src/google/protobuf/arenastring.h b/src/google/protobuf/arenastring.h index b8d31fec01..6bc8395f23 100644 --- a/src/google/protobuf/arenastring.h +++ b/src/google/protobuf/arenastring.h @@ -96,13 +96,12 @@ class PROTOBUF_EXPORT LazyString { class TaggedStringPtr { public: - // Bit flags qualifying string properties. We can use up to 3 bits as - // ptr_ is guaranteed and enforced to be aligned on 8 byte boundaries. + // Bit flags qualifying string properties. We can use 2 bits as + // ptr_ is guaranteed and enforced to be aligned on 4 byte boundaries. enum Flags { kArenaBit = 0x1, // ptr is arena allocated - kAllocatedBit = 0x2, // ptr is heap allocated - kMutableBit = 0x4, // ptr contents are fully mutable - kMask = 0x7 // Bit mask + kMutableBit = 0x2, // ptr contents are fully mutable + kMask = 0x3 // Bit mask }; // Composed logical types @@ -112,7 +111,7 @@ class TaggedStringPtr { // Allocated strings are mutable and (as the name implies) owned. // A heap allocated string must be deleted. - kAllocated = kAllocatedBit | kMutableBit, + kAllocated = kMutableBit, // Mutable arena strings are strings where the string instance is owned // by the arena, but the string contents itself are owned by the string @@ -166,8 +165,16 @@ class TaggedStringPtr { // Returns true if the current string is an immutable default value. inline bool IsDefault() const { return (as_int() & kMask) == kDefault; } - // Returns true if the current string is a heap allocated mutable value. - inline bool IsAllocated() const { return as_int() & kAllocatedBit; } + // If the current string is a heap-allocated mutable value, returns a pointer + // to it. Returns nullptr otherwise. + inline std::string *GetIfAllocated() const { + auto allocated = as_int() ^ kAllocated; + if (allocated & kMask) return nullptr; + + auto ptr = reinterpret_cast(allocated); + PROTOBUF_ASSUME(ptr != nullptr); + return ptr; + } // Returns true if the current string is an arena allocated value. // This means it's either a mutable or fixed size arena string. @@ -224,8 +231,8 @@ static_assert(std::is_trivial::value, // Because ArenaStringPtr is used in oneof unions, its constructor is a NOP and // the field is always manually initialized via method calls. // -// See TaggedPtr for more information about the types of string values being -// held, and the mutable and ownership invariants for each type. +// See TaggedStringPtr for more information about the types of string values +// being held, and the mutable and ownership invariants for each type. struct PROTOBUF_EXPORT ArenaStringPtr { ArenaStringPtr() = default; constexpr ArenaStringPtr(ExplicitlyConstructedArenaString* default_value, diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 7383e3eb3b..d78d9fed2f 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -61,12 +61,13 @@ PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased } // namespace -PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10; +PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = { + .next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10}; -ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); } +ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; -void ThreadSafeArenaStats::PrepareForSampling() { +void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { num_allocations.store(0, std::memory_order_relaxed); num_resets.store(0, std::memory_order_relaxed); bytes_requested.store(0, std::memory_order_relaxed); @@ -74,6 +75,7 @@ void ThreadSafeArenaStats::PrepareForSampling() { bytes_wasted.store(0, std::memory_order_relaxed); max_bytes_allocated.store(0, std::memory_order_relaxed); thread_ids.store(0, std::memory_order_relaxed); + weight = stride; // The inliner makes hardcoded skip_count difficult (especially when combined // with LTO). We use the ability to exclude stacks by regex when encoding // instead. @@ -105,12 +107,15 @@ void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested, info->thread_ids.fetch_or(tid, std::memory_order_relaxed); } -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { - bool first = *next_sample < 0; - *next_sample = g_exponential_biased_generator.GetStride( +ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) { + bool first = sampling_state.next_sample < 0; + const int64_t next_stride = g_exponential_biased_generator.GetStride( g_arenaz_sample_parameter.load(std::memory_order_relaxed)); // Small values of interval are equivalent to just sampling next time. - ABSL_ASSERT(*next_sample >= 1); + ABSL_ASSERT(next_stride >= 1); + sampling_state.next_sample = next_stride; + const int64_t old_stride = + absl::exchange(sampling_state.sample_stride, next_stride); // g_arenaz_enabled can be dynamically flipped, we need to set a threshold low // enough that we will start sampling in a reasonable time, so we just use the @@ -119,11 +124,11 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { // We will only be negative on our first count, so we should just retry in // that case. if (first) { - if (PROTOBUF_PREDICT_TRUE(--*next_sample > 0)) return nullptr; - return SampleSlow(next_sample); + if (PROTOBUF_PREDICT_TRUE(--sampling_state.next_sample > 0)) return nullptr; + return SampleSlow(sampling_state); } - return GlobalThreadSafeArenazSampler().Register(); + return GlobalThreadSafeArenazSampler().Register(old_stride); } void SetThreadSafeArenazEnabled(bool enabled) { @@ -150,7 +155,8 @@ void SetThreadSafeArenazMaxSamples(int32_t max) { void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) { if (next_sample >= 0) { - global_next_sample = next_sample; + global_sampling_state.next_sample = next_sample; + global_sampling_state.sample_stride = next_sample; } else { ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz next sample: %lld", static_cast(next_sample)); // NOLINT(runtime/int) diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index 76698a26a3..31c178a009 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -58,8 +58,11 @@ struct ThreadSafeArenaStats ~ThreadSafeArenaStats(); // Puts the object into a clean state, fills in the logically `const` members, - // blocking for any readers that are currently sampling the object. - void PrepareForSampling() ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); + // blocking for any readers that are currently sampling the object. The + // 'stride' parameter is the number of ThreadSafeArenas that were instantiated + // between this sample and the previous one. + void PrepareForSampling(int64_t stride) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); // These fields are mutated by the various Record* APIs and need to be // thread-safe. @@ -91,7 +94,18 @@ struct ThreadSafeArenaStats } }; -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample); +struct SamplingState { + // Number of ThreadSafeArenas that should be instantiated before the next + // ThreadSafeArena is sampled. This variable is decremented with each + // instantiation. + int64_t next_sample; + // When we make a sampling decision, we record that distance between from the + // previous sample so we can weight each sample. 'distance' here is the + // number of instantiations of ThreadSafeArena. + int64_t sample_stride; +}; + +ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state); void UnsampleSlow(ThreadSafeArenaStats* info); class ThreadSafeArenaStatsHandle { @@ -138,24 +152,27 @@ class ThreadSafeArenaStatsHandle { using ThreadSafeArenazSampler = ::absl::profiling_internal::SampleRecorder; -extern PROTOBUF_THREAD_LOCAL int64_t global_next_sample; +extern PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state; // Returns an RAII sampling handle that manages registration and unregistation // with the global sampler. inline ThreadSafeArenaStatsHandle Sample() { - if (PROTOBUF_PREDICT_TRUE(--global_next_sample > 0)) { + if (PROTOBUF_PREDICT_TRUE(--global_sampling_state.next_sample > 0)) { return ThreadSafeArenaStatsHandle(nullptr); } - return ThreadSafeArenaStatsHandle(SampleSlow(&global_next_sample)); + return ThreadSafeArenaStatsHandle(SampleSlow(global_sampling_state)); } #else + +using SamplingState = int64_t; + struct ThreadSafeArenaStats { static void RecordAllocateStats(ThreadSafeArenaStats*, size_t /*requested*/, size_t /*allocated*/, size_t /*wasted*/) {} }; -ThreadSafeArenaStats* SampleSlow(int64_t* next_sample); +ThreadSafeArenaStats* SampleSlow(SamplingState& next_sample); void UnsampleSlow(ThreadSafeArenaStats* info); class ThreadSafeArenaStatsHandle { diff --git a/src/google/protobuf/arenaz_sampler_test.cc b/src/google/protobuf/arenaz_sampler_test.cc index 1bfec542d6..2588c04a75 100644 --- a/src/google/protobuf/arenaz_sampler_test.cc +++ b/src/google/protobuf/arenaz_sampler_test.cc @@ -64,8 +64,9 @@ std::vector GetBytesAllocated(ThreadSafeArenazSampler* s) { return res; } -ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size) { - auto* info = s->Register(); +ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size, + int64_t stride) { + auto* info = s->Register(stride); assert(info != nullptr); info->bytes_allocated.store(size); return info; @@ -79,8 +80,9 @@ namespace { TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 107; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); EXPECT_EQ(info.num_allocations.load(), 0); EXPECT_EQ(info.num_resets.load(), 0); @@ -88,6 +90,7 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { EXPECT_EQ(info.bytes_allocated.load(), 0); EXPECT_EQ(info.bytes_wasted.load(), 0); EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.weight, kTestStride); info.num_allocations.store(1, std::memory_order_relaxed); info.num_resets.store(1, std::memory_order_relaxed); @@ -96,19 +99,21 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { info.bytes_wasted.store(1, std::memory_order_relaxed); info.max_bytes_allocated.store(1, std::memory_order_relaxed); - info.PrepareForSampling(); + info.PrepareForSampling(2 * kTestStride); EXPECT_EQ(info.num_allocations.load(), 0); EXPECT_EQ(info.num_resets.load(), 0); EXPECT_EQ(info.bytes_requested.load(), 0); EXPECT_EQ(info.bytes_allocated.load(), 0); EXPECT_EQ(info.bytes_wasted.load(), 0); EXPECT_EQ(info.max_bytes_allocated.load(), 0); + EXPECT_EQ(info.weight, 2 * kTestStride); } TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 458; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); EXPECT_EQ(info.num_allocations.load(), 1); EXPECT_EQ(info.num_resets.load(), 0); @@ -128,8 +133,9 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { TEST(ThreadSafeArenaStatsTest, RecordResetSlow) { ThreadSafeArenaStats info; + constexpr int64_t kTestStride = 584; MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(kTestStride); EXPECT_EQ(info.num_resets.load(), 0); EXPECT_EQ(info.bytes_allocated.load(), 0); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); @@ -143,11 +149,12 @@ TEST(ThreadSafeArenaStatsTest, RecordResetSlow) { TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) { SetThreadSafeArenazEnabled(true); SetThreadSafeArenazSampleParameter(100); + constexpr int64_t kTestStride = 0; for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - ThreadSafeArenaStats* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState sampling_state = {kTestStride, kTestStride}; + ThreadSafeArenaStats* sample = SampleSlow(sampling_state); + EXPECT_GT(sampling_state.next_sample, 0); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } @@ -156,11 +163,12 @@ TEST(ThreadSafeArenazSamplerTest, SmallSampleParameter) { TEST(ThreadSafeArenazSamplerTest, LargeSampleParameter) { SetThreadSafeArenazEnabled(true); SetThreadSafeArenazSampleParameter(std::numeric_limits::max()); + constexpr int64_t kTestStride = 0; for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - ThreadSafeArenaStats* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState sampling_state = {kTestStride, kTestStride}; + ThreadSafeArenaStats* sample = SampleSlow(sampling_state); + EXPECT_GT(sampling_state.next_sample, 0); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } @@ -187,7 +195,8 @@ TEST(ThreadSafeArenazSamplerTest, Sample) { TEST(ThreadSafeArenazSamplerTest, Handle) { auto& sampler = GlobalThreadSafeArenazSampler(); - ThreadSafeArenaStatsHandle h(sampler.Register()); + constexpr int64_t kTestStride = 17; + ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride)); auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h); info->bytes_allocated.store(0x12345678, std::memory_order_relaxed); @@ -195,6 +204,7 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { sampler.Iterate([&](const ThreadSafeArenaStats& h) { if (&h == info) { EXPECT_EQ(h.bytes_allocated.load(), 0x12345678); + EXPECT_EQ(h.weight, kTestStride); found = true; } }); @@ -216,10 +226,11 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { TEST(ThreadSafeArenazSamplerTest, Registration) { ThreadSafeArenazSampler sampler; - auto* info1 = Register(&sampler, 1); + constexpr int64_t kTestStride = 100; + auto* info1 = Register(&sampler, 1, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1)); - auto* info2 = Register(&sampler, 2); + auto* info2 = Register(&sampler, 2, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2)); info1->bytes_allocated.store(3); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2)); @@ -231,16 +242,17 @@ TEST(ThreadSafeArenazSamplerTest, Registration) { TEST(ThreadSafeArenazSamplerTest, Unregistration) { ThreadSafeArenazSampler sampler; std::vector infos; + constexpr int64_t kTestStride = 200; for (size_t i = 0; i < 3; ++i) { - infos.push_back(Register(&sampler, i)); + infos.push_back(Register(&sampler, i, kTestStride)); } EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2)); sampler.Unregister(infos[1]); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2)); - infos.push_back(Register(&sampler, 3)); - infos.push_back(Register(&sampler, 4)); + infos.push_back(Register(&sampler, 3, kTestStride)); + infos.push_back(Register(&sampler, 4, kTestStride)); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4)); sampler.Unregister(infos[3]); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4)); @@ -257,18 +269,19 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { ThreadPool pool(10); for (int i = 0; i < 10; ++i) { - pool.Schedule([&sampler, &stop]() { + const int64_t sampling_stride = 11 + i % 3; + pool.Schedule([&sampler, &stop, sampling_stride]() { std::random_device rd; std::mt19937 gen(rd()); std::vector infoz; while (!stop.HasBeenNotified()) { if (infoz.empty()) { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride)); } switch (std::uniform_int_distribution<>(0, 1)(gen)) { case 0: { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride)); break; } case 1: { @@ -277,6 +290,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { ThreadSafeArenaStats* info = infoz[p]; infoz[p] = infoz.back(); infoz.pop_back(); + EXPECT_EQ(info->weight, sampling_stride); sampler.Unregister(info); break; } @@ -292,9 +306,10 @@ TEST(ThreadSafeArenazSamplerTest, MultiThreaded) { TEST(ThreadSafeArenazSamplerTest, Callback) { ThreadSafeArenazSampler sampler; + constexpr int64_t kTestStride = 203; - auto* info1 = Register(&sampler, 1); - auto* info2 = Register(&sampler, 2); + auto* info1 = Register(&sampler, 1, kTestStride); + auto* info2 = Register(&sampler, 2, kTestStride); static const ThreadSafeArenaStats* expected; diff --git a/src/google/protobuf/compiler/cpp/enum_field.cc b/src/google/protobuf/compiler/cpp/enum_field.cc index 8ffb699e87..3539a0df41 100644 --- a/src/google/protobuf/compiler/cpp/enum_field.cc +++ b/src/google/protobuf/compiler/cpp/enum_field.cc @@ -252,7 +252,9 @@ void RepeatedEnumFieldGenerator::GeneratePrivateMembers( format("::$proto_ns$::RepeatedField $name$_;\n"); if (descriptor_->is_packed() && HasGeneratedMethods(descriptor_->file(), options_)) { - format("mutable std::atomic $cached_byte_size_name$;\n"); + format( + "mutable ::$proto_ns$::internal::CachedSize " + "$cached_byte_size_name$;\n"); } } @@ -364,7 +366,7 @@ void RepeatedEnumFieldGenerator::GenerateSerializeWithCachedSizesToArray( format( "{\n" " int byte_size = " - "$cached_byte_size_field$.load(std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Get();\n" " if (byte_size > 0) {\n" " target = stream->WriteEnumPacked(\n" " $number$, $field$, byte_size, target);\n" @@ -402,8 +404,7 @@ void RepeatedEnumFieldGenerator::GenerateByteSize(io::Printer* printer) const { "::_pbi::WireFormatLite::Int32Size(static_cast<$int32$>(data_size));\n" "}\n" "int cached_size = ::_pbi::ToCachedSize(data_size);\n" - "$cached_byte_size_field$.store(cached_size,\n" - " std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Set(cached_size);\n" "total_size += data_size;\n"); } else { format("total_size += ($tag_size$UL * count) + data_size;\n"); diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index f4eb744e2f..af463d58ac 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -2101,6 +2101,11 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { "static constexpr int32_t kHasBitsOffset =\n" " 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n"); } + if (descriptor_->real_oneof_decl_count() > 0) { + format( + "static constexpr int32_t kOneofCaseOffset =\n" + " PROTOBUF_FIELD_OFFSET($classtype$, $oneof_case$);\n"); + } for (auto field : FieldRange(descriptor_)) { field_generators_.get(field).GenerateInternalAccessorDeclarations(printer); if (IsFieldStripped(field, options_)) { diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 3254b37581..db4d209777 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -292,23 +292,11 @@ TailCallTableInfo::TailCallTableInfo( const std::vector& has_bit_indices, const std::vector& inlined_string_indices, MessageSCCAnalyzer* scc_analyzer) { - int oneof_count = descriptor->real_oneof_decl_count(); - // If this message has any oneof fields, store the case offset in the first - // auxiliary entry. - if (oneof_count > 0) { - GOOGLE_LOG_IF(DFATAL, ordered_fields.empty()) - << "Invalid message: " << descriptor->full_name() << " has " - << oneof_count << " oneof declarations, but no fields"; - aux_entries.push_back(StrCat("_fl::Offset{offsetof(", - ClassName(descriptor), - ", _impl_._oneof_case_)}")); - } - // If this message has any inlined string fields, store the donation state // offset in the second auxiliary entry. if (!inlined_string_indices.empty()) { - aux_entries.resize(2); // pad if necessary - aux_entries[1] = + aux_entries.resize(1); // pad if necessary + aux_entries[0] = StrCat("_fl::Offset{offsetof(", ClassName(descriptor), ", _impl_._inlined_string_donated_)}"); } @@ -1102,7 +1090,7 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) { FieldMemberName(field, /*cold=*/false)); } if (oneof) { - format("$1$, ", oneof->index()); + format("_Internal::kOneofCaseOffset + $1$, ", 4 * oneof->index()); } else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) { if (entry.hasbit_idx >= 0) { format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx); diff --git a/src/google/protobuf/compiler/cpp/primitive_field.cc b/src/google/protobuf/compiler/cpp/primitive_field.cc index 6c92ede251..342a3b0731 100644 --- a/src/google/protobuf/compiler/cpp/primitive_field.cc +++ b/src/google/protobuf/compiler/cpp/primitive_field.cc @@ -328,7 +328,9 @@ void RepeatedPrimitiveFieldGenerator::GeneratePrivateMembers( format("::$proto_ns$::RepeatedField< $type$ > $name$_;\n"); if (descriptor_->is_packed() && FixedSize(descriptor_->type()) == -1 && HasGeneratedMethods(descriptor_->file(), options_)) { - format("mutable std::atomic $cached_byte_size_name$;\n"); + format( + "mutable ::$proto_ns$::internal::CachedSize " + "$cached_byte_size_name$;\n"); } } @@ -433,7 +435,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateSerializeWithCachedSizesToArray( format( "{\n" " int byte_size = " - "$cached_byte_size_field$.load(std::memory_order_relaxed);\n" + "$cached_byte_size_field$.Get();\n" " if (byte_size > 0) {\n" " target = stream->Write$declared_type$Packed(\n" " $number$, _internal_$name$(), byte_size, target);\n" @@ -484,8 +486,7 @@ void RepeatedPrimitiveFieldGenerator::GenerateByteSize( if (FixedSize(descriptor_->type()) == -1) { format( "int cached_size = ::_pbi::ToCachedSize(data_size);\n" - "$cached_byte_size_field$.store(cached_size,\n" - " std::memory_order_relaxed);\n"); + "$cached_byte_size_field$.Set(cached_size);\n"); } format("total_size += data_size;\n"); } else { diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 619d71518a..5e61898c40 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -31,6 +31,7 @@ cc_library( "message_field.cc", "message_field_lite.cc", "message_lite.cc", + "message_serialization.cc", "name_resolver.cc", "primitive_field.cc", "primitive_field_lite.cc", @@ -62,6 +63,7 @@ cc_library( "message_field.h", "message_field_lite.h", "message_lite.h", + "message_serialization.h", "name_resolver.h", "names.h", "options.h", diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index 95917a7f6d..5adacef3ad 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -53,6 +53,7 @@ #include #include #include +#include #include #include @@ -587,13 +588,6 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( std::unique_ptr sorted_fields( SortFieldsByNumber(descriptor_)); - std::vector sorted_extensions; - sorted_extensions.reserve(descriptor_->extension_range_count()); - for (int i = 0; i < descriptor_->extension_range_count(); ++i) { - sorted_extensions.push_back(descriptor_->extension_range(i)); - } - std::sort(sorted_extensions.begin(), sorted_extensions.end(), - ExtensionRangeOrdering()); printer->Print( "@java.lang.Override\n" "public void writeTo(com.google.protobuf.CodedOutputStream output)\n" @@ -628,19 +622,8 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( } } - // Merge the fields and the extension ranges, both sorted by field number. - for (int i = 0, j = 0; - i < descriptor_->field_count() || j < sorted_extensions.size();) { - if (i == descriptor_->field_count()) { - GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]); - } else if (j == sorted_extensions.size()) { - GenerateSerializeOneField(printer, sorted_fields[i++]); - } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) { - GenerateSerializeOneField(printer, sorted_fields[i++]); - } else { - GenerateSerializeOneExtensionRange(printer, sorted_extensions[j++]); - } - } + GenerateSerializeFieldsAndExtensions(printer, field_generators_, descriptor_, + sorted_fields.get()); if (descriptor_->options().message_set_wire_format()) { printer->Print("unknownFields.writeAsMessageSetTo(output);\n"); @@ -770,17 +753,6 @@ void ImmutableMessageGenerator::GenerateParseFromMethods(io::Printer* printer) { GeneratedCodeVersionSuffix()); } -void ImmutableMessageGenerator::GenerateSerializeOneField( - io::Printer* printer, const FieldDescriptor* field) { - field_generators_.get(field).GenerateSerializationCode(printer); -} - -void ImmutableMessageGenerator::GenerateSerializeOneExtensionRange( - io::Printer* printer, const Descriptor::ExtensionRange* range) { - printer->Print("extensionWriter.writeUntil($end$, output);\n", "end", - StrCat(range->end)); -} - // =================================================================== void ImmutableMessageGenerator::GenerateBuilder(io::Printer* printer) { @@ -1017,8 +989,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( " return true;\n" "}\n" "if (!(obj instanceof $classname$)) {\n" - // don't simply return false because mutable and immutable types - // can be equal + // don't simply return false because mutable and immutable types + // can be equal " return super.equals(obj);\n" "}\n" "$classname$ other = ($classname$) obj;\n" diff --git a/src/google/protobuf/compiler/java/message.h b/src/google/protobuf/compiler/java/message.h index 2dbd0dd9bc..45da8ff646 100644 --- a/src/google/protobuf/compiler/java/message.h +++ b/src/google/protobuf/compiler/java/message.h @@ -123,10 +123,6 @@ class ImmutableMessageGenerator : public MessageGenerator { void GenerateMessageSerializationMethods(io::Printer* printer); void GenerateParseFromMethods(io::Printer* printer); - void GenerateSerializeOneField(io::Printer* printer, - const FieldDescriptor* field); - void GenerateSerializeOneExtensionRange( - io::Printer* printer, const Descriptor::ExtensionRange* range); void GenerateBuilder(io::Printer* printer); void GenerateIsInitialized(io::Printer* printer); diff --git a/src/google/protobuf/compiler/java/message_serialization.cc b/src/google/protobuf/compiler/java/message_serialization.cc new file mode 100644 index 0000000000..5db627d322 --- /dev/null +++ b/src/google/protobuf/compiler/java/message_serialization.cc @@ -0,0 +1,50 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +void GenerateSerializeExtensionRange(io::Printer* printer, + const Descriptor::ExtensionRange* range) { + printer->Print("extensionWriter.writeUntil($end$, output);\n", "end", + StrCat(range->end)); +} + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/java/message_serialization.h b/src/google/protobuf/compiler/java/message_serialization.h new file mode 100644 index 0000000000..6145392f81 --- /dev/null +++ b/src/google/protobuf/compiler/java/message_serialization.h @@ -0,0 +1,91 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ + +#include +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +// Generates code to serialize a single extension range. +void GenerateSerializeExtensionRange(io::Printer* printer, + const Descriptor::ExtensionRange* range); + +// Generates code to serialize all fields and extension ranges for the specified +// message descriptor, sorting serialization calls in increasing order by field +// number. +// +// Templatized to support different field generator implementations. +template +void GenerateSerializeFieldsAndExtensions( + io::Printer* printer, + const FieldGeneratorMap& field_generators, + const Descriptor* descriptor, const FieldDescriptor** sorted_fields) { + std::vector sorted_extensions; + sorted_extensions.reserve(descriptor->extension_range_count()); + for (int i = 0; i < descriptor->extension_range_count(); ++i) { + sorted_extensions.push_back(descriptor->extension_range(i)); + } + std::sort(sorted_extensions.begin(), sorted_extensions.end(), + ExtensionRangeOrdering()); + + // Merge the fields and the extension ranges, both sorted by field number. + for (int i = 0, j = 0; + i < descriptor->field_count() || j < sorted_extensions.size();) { + if (i == descriptor->field_count()) { + GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); + } else if (j == sorted_extensions.size()) { + field_generators.get(sorted_fields[i++]) + .GenerateSerializationCode(printer); + } else if (sorted_fields[i]->number() < sorted_extensions[j]->start) { + field_generators.get(sorted_fields[i++]) + .GenerateSerializationCode(printer); + } else { + GenerateSerializeExtensionRange(printer, sorted_extensions[j++]); + } + } +} + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_MESSAGE_SERIALIZATION_H__ diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 257798e4ae..c77c83c1e9 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -10437,7 +10437,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize( // repeated int32 path = 1 [packed = true]; { - int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._path_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 1, _internal_path(), byte_size, target); @@ -10446,7 +10446,7 @@ uint8_t* SourceCodeInfo_Location::_InternalSerialize( // repeated int32 span = 2 [packed = true]; { - int byte_size = _impl_._span_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._span_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 2, _internal_span(), byte_size, target); @@ -10509,8 +10509,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._path_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._path_cached_byte_size_.Set(cached_size); total_size += data_size; } @@ -10523,8 +10522,7 @@ size_t SourceCodeInfo_Location::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._span_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._span_cached_byte_size_.Set(cached_size); total_size += data_size; } @@ -10996,7 +10994,7 @@ uint8_t* GeneratedCodeInfo_Annotation::_InternalSerialize( // repeated int32 path = 1 [packed = true]; { - int byte_size = _impl_._path_cached_byte_size_.load(std::memory_order_relaxed); + int byte_size = _impl_._path_cached_byte_size_.Get(); if (byte_size > 0) { target = stream->WriteInt32Packed( 1, _internal_path(), byte_size, target); @@ -11051,8 +11049,7 @@ size_t GeneratedCodeInfo_Annotation::ByteSizeLong() const { ::_pbi::WireFormatLite::Int32Size(static_cast(data_size)); } int cached_size = ::_pbi::ToCachedSize(data_size); - _impl_._path_cached_byte_size_.store(cached_size, - std::memory_order_relaxed); + _impl_._path_cached_byte_size_.Set(cached_size); total_size += data_size; } diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index b8710b274c..84e40775ea 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -7962,9 +7962,9 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_; - mutable std::atomic _path_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > span_; - mutable std::atomic _span_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _span_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField leading_detached_comments_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr leading_comments_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr trailing_comments_; @@ -8350,7 +8350,7 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final : ::PROTOBUF_NAMESPACE_ID::internal::HasBits<1> _has_bits_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< int32_t > path_; - mutable std::atomic _path_cached_byte_size_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _path_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr source_file_; int32_t begin_; int32_t end_; diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 713114ec4f..8e4036974e 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -454,30 +454,36 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } - auto saved_tag = UnalignedLoad(ptr); - ptr += sizeof(TagType); - SyncHasbits(msg, hasbits, table); - auto* aux = table->field_aux(data.aux_idx()); - if (aux_is_table) { - auto* inner_table = aux->table; - auto& field = RefAt(msg, data.offset()); + const auto expected_tag = UnalignedLoad(ptr); + const auto aux = *table->field_aux(data.aux_idx()); + auto& field = RefAt(msg, data.offset()); + do { + ptr += sizeof(TagType); MessageLite* submsg = field.Add>( - inner_table->default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag), - inner_table); + aux_is_table ? aux.table->default_instance : aux.message_default()); + if (aux_is_table) { + if (group_coding) { + ptr = ctx->ParseGroup(submsg, ptr, + FastDecodeTag(expected_tag), aux.table); + } else { + ptr = ctx->ParseMessage(submsg, ptr, aux.table); + } + } else { + if (group_coding) { + ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag)); + } else { + ptr = ctx->ParseMessage(submsg, ptr); + } } - return ctx->ParseMessage(submsg, ptr, inner_table); - } else { - const MessageLite* default_instance = aux->message_default(); - auto& field = RefAt(msg, data.offset()); - MessageLite* submsg = - field.Add>(default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { + PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_PASS); } - return ctx->ParseMessage(submsg, ptr); - } + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) { + PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_PASS); + } + } while (UnalignedLoad(ptr) == expected_tag); + + PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_PASS); } const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { @@ -1351,11 +1357,8 @@ bool TcParser::ChangeOneof(const TcParseTableBase* table, const TcParseTableBase::FieldEntry& entry, uint32_t field_num, ParseContext* ctx, MessageLite* msg) { - // The _oneof_case_ array offset is stored in the first aux entry. - uint32_t oneof_case_offset = table->field_aux(0u)->offset; - // The _oneof_case_ array index is stored in the has-bit index. - uint32_t* oneof_case = - &TcParser::RefAt(msg, oneof_case_offset) + entry.has_idx; + // The _oneof_case_ value offset is stored in the has-bit index. + uint32_t* oneof_case = &TcParser::RefAt(msg, entry.has_idx); uint32_t current_case = *oneof_case; *oneof_case = field_num; diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index 71d15cdebc..149986d92a 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -185,13 +185,58 @@ T* GetOwnedMessage(Arena* message_arena, T* submessage, // Hide atomic from the public header and allow easy change to regular int // on platforms where the atomic might have a perf impact. +// +// CachedSize is like std::atomic but with some important changes: +// +// 1) CachedSize uses Get / Set rather than load / store. +// 2) CachedSize always uses relaxed ordering. +// 3) CachedSize is assignable and copy-constructible. +// 4) CachedSize has a constexpr default constructor, and a constexpr +// constructor that takes an int argument. +// 5) If the compiler supports the __atomic_load_n / __atomic_store_n builtins, +// then CachedSize is trivially copyable. +// +// Developed at https://godbolt.org/z/vYcx7zYs1 ; supports gcc, clang, MSVC. class PROTOBUF_EXPORT CachedSize { + private: + using Scalar = int; + public: - int Get() const { return size_.load(std::memory_order_relaxed); } - void Set(int size) { size_.store(size, std::memory_order_relaxed); } + constexpr CachedSize() noexcept : atom_(Scalar{}) {} + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr CachedSize(Scalar desired) noexcept : atom_(desired) {} +#if PROTOBUF_BUILTIN_ATOMIC + constexpr CachedSize(const CachedSize& other) = default; + + Scalar Get() const noexcept { + return __atomic_load_n(&atom_, __ATOMIC_RELAXED); + } + + void Set(Scalar desired) noexcept { + __atomic_store_n(&atom_, desired, __ATOMIC_RELAXED); + } +#else + CachedSize(const CachedSize& other) noexcept : atom_(other.Get()) {} + CachedSize& operator=(const CachedSize& other) noexcept { + Set(other.Get()); + return *this; + } + + Scalar Get() const noexcept { // + return atom_.load(std::memory_order_relaxed); + } + + void Set(Scalar desired) noexcept { + atom_.store(desired, std::memory_order_relaxed); + } +#endif private: - std::atomic size_{0}; +#if PROTOBUF_BUILTIN_ATOMIC + Scalar atom_; +#else + std::atomic atom_; +#endif }; PROTOBUF_EXPORT void DestroyMessage(const void* message); diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 0b463b8c6a..1da0410fb9 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -139,6 +139,11 @@ #define PROTOBUF_BUILTIN_BSWAP64(x) __builtin_bswap64(x) #endif +// Portable check for gcc-style atomic built-ins +#if __has_builtin(__atomic_load_n) +#define PROTOBUF_BUILTIN_ATOMIC 1 +#endif + // Portable check for GCC minimum version: // https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html #if defined(__GNUC__) && defined(__GNUC_MINOR__) \ diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 44663219f0..184980a7e9 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -38,6 +38,7 @@ #undef PROTOBUF_BUILTIN_BSWAP16 #undef PROTOBUF_BUILTIN_BSWAP32 #undef PROTOBUF_BUILTIN_BSWAP64 +#undef PROTOBUF_BUILTIN_ATOMIC #undef PROTOBUF_GNUC_MIN #undef PROTOBUF_MSC_VER_MIN #undef PROTOBUF_CPLUSPLUS_MIN diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 43c1eae0a3..e746616fb7 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -419,6 +419,8 @@ void Struct::InternalSwap(Struct* other) { class Value::_Internal { public: + static constexpr int32_t kOneofCaseOffset = + PROTOBUF_FIELD_OFFSET(::PROTOBUF_NAMESPACE_ID::Value, _impl_._oneof_case_); static const ::PROTOBUF_NAMESPACE_ID::Struct& struct_value(const Value* msg); static const ::PROTOBUF_NAMESPACE_ID::ListValue& list_value(const Value* msg); }; diff --git a/src/google/protobuf/stubs/status.cc b/src/google/protobuf/stubs/status.cc index f5c0fa48f1..fc6c7bab0d 100644 --- a/src/google/protobuf/stubs/status.cc +++ b/src/google/protobuf/stubs/status.cc @@ -28,9 +28,10 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include +#include +#include #include -#include #include #include @@ -256,6 +257,12 @@ Status UnknownError(StringPiece message) { return Status(StatusCode::kUnknown, message); } +Status ErrnoToStatus(int error_number, StringPiece message) { + // We will take an Abseil dependency soon, so no reason to do anything + // elaborate here. + return InternalError(StrCat(message, ": ", strerror(error_number))); +} + } // namespace status_internal } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/stubs/status.h b/src/google/protobuf/stubs/status.h index c858cf6239..919e54ae2e 100644 --- a/src/google/protobuf/stubs/status.h +++ b/src/google/protobuf/stubs/status.h @@ -31,10 +31,12 @@ #ifndef GOOGLE_PROTOBUF_STUBS_STATUS_H_ #define GOOGLE_PROTOBUF_STUBS_STATUS_H_ -#include - #include +#include + +#include +// Must be included last. #include namespace google { @@ -147,6 +149,8 @@ PROTOBUF_EXPORT Status UnavailableError(StringPiece message); PROTOBUF_EXPORT Status UnimplementedError(StringPiece message); PROTOBUF_EXPORT Status UnknownError(StringPiece message); +PROTOBUF_EXPORT Status ErrnoToStatus(int error_number, StringPiece message); + } // namespace status_internal using ::google::protobuf::util::status_internal::Status; @@ -187,6 +191,8 @@ using ::google::protobuf::util::status_internal::UnavailableError; using ::google::protobuf::util::status_internal::UnimplementedError; using ::google::protobuf::util::status_internal::UnknownError; +using ::google::protobuf::util::status_internal::ErrnoToStatus; + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index afa5e2e474..777d060bbd 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -73,7 +73,6 @@ ProtoWriter::ProtoWriter(TypeResolver* type_resolver, element_(nullptr), size_insert_(), output_(output), - buffer_(), adapter_(&buffer_), stream_(new CodedOutputStream(&adapter_)), listener_(listener), @@ -95,7 +94,6 @@ ProtoWriter::ProtoWriter(const TypeInfo* typeinfo, element_(nullptr), size_insert_(), output_(output), - buffer_(), adapter_(&buffer_), stream_(new CodedOutputStream(&adapter_)), listener_(listener), diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index a8b68a3c76..29ffe865d9 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -84,7 +84,7 @@ util::Status GetStatus(const util::StatusOr& s) { } MATCHER_P(StatusIs, status, - StrCat(".status() is ", testing::PrintToString(status))) { + StrCat(".status() is ", testing::PrintToString(status))) { return GetStatus(arg).code() == status; } @@ -412,6 +412,110 @@ TEST(JsonUtilTest, TestDynamicMessage) { EXPECT_EQ(*message_json, *generated_json); } +TEST(JsonUtilTest, TestParsingAny) { + StringPiece input = R"json( + { + "value": { + "@type": "type.googleapis.com/proto3.TestMessage", + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1} + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); + + EXPECT_THAT( + ToJson(m), + IsOkAndHolds( + R"json({"value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}})json")); +} + +TEST(JsonUtilTest, TestParsingAnyMiddleAtType) { + StringPiece input = R"json( + { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "@type": "type.googleapis.com/proto3.TestMessage", + "message_value": {"value": 1} + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); +} + +TEST(JsonUtilTest, TestParsingAnyEndAtType) { + StringPiece input = R"json( + { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1}, + "@type": "type.googleapis.com/proto3.TestMessage" + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + TestMessage t; + EXPECT_TRUE(m.value().UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); +} + +TEST(JsonUtilTest, TestParsingNestedAnys) { + StringPiece input = R"json( + { + "value": { + "value": { + "int32_value": 5, + "string_value": "expected_value", + "message_value": {"value": 1}, + "@type": "type.googleapis.com/proto3.TestMessage" + }, + "@type": "type.googleapis.com/google.protobuf.Any" + } + } + )json"; + + TestAny m; + ASSERT_OK(FromJson(input, &m)); + + google::protobuf::Any inner; + EXPECT_TRUE(m.value().UnpackTo(&inner)); + + TestMessage t; + EXPECT_TRUE(inner.UnpackTo(&t)); + EXPECT_EQ(t.int32_value(), 5); + EXPECT_EQ(t.string_value(), "expected_value"); + EXPECT_EQ(t.message_value().value(), 1); + + EXPECT_THAT( + ToJson(m), + IsOkAndHolds( + R"json({"value":{"@type":"type.googleapis.com/google.protobuf.Any","value":{"@type":"type.googleapis.com/proto3.TestMessage","int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})json")); +} + TEST(JsonUtilTest, TestParsingUnknownAnyFields) { StringPiece input = R"json( { @@ -672,9 +776,9 @@ TEST(JsonUtilTest, TestWrongJsonInput) { auto* resolver = NewTypeResolverForDescriptorPool( "type.googleapis.com", DescriptorPool::generated_pool()); - EXPECT_THAT(JsonToBinaryStream(resolver, message_type, &input_stream, - &output_stream), - StatusIs(util::StatusCode::kInvalidArgument)); + EXPECT_THAT( + JsonToBinaryStream(resolver, message_type, &input_stream, &output_stream), + StatusIs(util::StatusCode::kInvalidArgument)); delete resolver; }