From 5987c896fd162381b9c9e219f6c124a1fc7a5a52 Mon Sep 17 00:00:00 2001 From: Cong Liu Date: Wed, 8 Nov 2023 18:17:04 -0800 Subject: [PATCH] Internal change. PiperOrigin-RevId: 580726494 --- src/google/protobuf/compiler/cpp/helpers.cc | 7 ++ src/google/protobuf/compiler/cpp/helpers.h | 5 + .../compiler/cpp/parse_function_generator.cc | 24 ++++- .../compiler/java/java_features.pb.cc | 3 + src/google/protobuf/compiler/plugin.pb.cc | 12 +++ src/google/protobuf/cpp_features.pb.cc | 3 + src/google/protobuf/descriptor.pb.cc | 96 +++++++++++++++++++ .../golden/compare_cpp_codegen_failure.txt | 4 +- .../golden/compare_cpp_codegen_failure.xml | 2 +- .../protobuf/generated_message_reflection.cc | 12 +++ .../protobuf/generated_message_tctable_decl.h | 34 +++++-- src/google/protobuf/port_def.inc | 6 ++ src/google/protobuf/port_undef.inc | 1 + 13 files changed, 196 insertions(+), 13 deletions(-) diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index c14e064c57..8a4fd03f72 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -1024,6 +1024,13 @@ bool IsPresentMessage(const Descriptor* descriptor, const Options& options) { return true; } +const FieldDescriptor* FindHottestField( + const std::vector& fields, const Options& options) { + (void)fields; + (void)options; + return nullptr; +} + static bool HasRepeatedFields(const Descriptor* descriptor) { for (int i = 0; i < descriptor->field_count(); ++i) { if (descriptor->field(i)->label() == FieldDescriptor::LABEL_REPEATED) { diff --git a/src/google/protobuf/compiler/cpp/helpers.h b/src/google/protobuf/compiler/cpp/helpers.h index 4c590cdc76..9275bf1f4f 100644 --- a/src/google/protobuf/compiler/cpp/helpers.h +++ b/src/google/protobuf/compiler/cpp/helpers.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" @@ -429,6 +430,10 @@ bool ShouldForceAllocationOnConstruction(const Descriptor* desc, // Returns true if the message is present based on PDProto profile. bool IsPresentMessage(const Descriptor* descriptor, const Options& options); +// Returns the most likely present field. Returns nullptr if not profile driven. +const FieldDescriptor* FindHottestField( + const std::vector& fields, const Options& options); + // Does the file contain any definitions that need extension_set.h? bool HasExtensionsOrExtendableMessage(const FileDescriptor* file); diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 25bfbbed03..3f511116d4 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -401,9 +401,29 @@ void ParseFunctionGenerator::GenerateTailCallTable(io::Printer* printer) { } format( "&$1$._instance,\n" - "$2$, // fallback\n" - "", + "$2$, // fallback\n", DefaultInstanceName(descriptor_, options_), fallback); + std::vector subtable_fields; + for (const auto& aux : tc_table_info_->aux_entries) { + if (aux.type == internal::TailCallTableInfo::kSubTable) { + subtable_fields.push_back(aux.field); + } + } + const auto* hottest = FindHottestField(subtable_fields, options_); + // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid + // branches. Set the pointer to itself to avoid nullptr. + printer->Emit( + {{"hottest_type_name", + QualifiedClassName( + hottest == nullptr ? descriptor_ : hottest->message_type(), + options_)}}, + // clang-format off + R"cc( +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE +::_pbi::TcParser::GetTable<$hottest_type_name$>(), // to_prefetch +#endif // PROTOBUF_PREFETCH_PARSE_TABLE + )cc"); + // clang-format on } format("}, {{\n"); { diff --git a/src/google/protobuf/compiler/java/java_features.pb.cc b/src/google/protobuf/compiler/java/java_features.pb.cc index df3c83a3b3..d1641d67f9 100644 --- a/src/google/protobuf/compiler/java/java_features.pb.cc +++ b/src/google/protobuf/compiler/java/java_features.pb.cc @@ -245,6 +245,9 @@ const ::_pbi::TcParseTable<1, 2, 1, 0, 2> JavaFeatures::_table_ = { offsetof(decltype(_table_), aux_entries), &_JavaFeatures_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::pb::JavaFeatures>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional .pb.JavaFeatures.Utf8Validation utf8_validation = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { {::_pbi::TcParser::FastEr0S1, diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index 17247abad6..45beeb2a03 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -433,6 +433,9 @@ const ::_pbi::TcParseTable<2, 4, 0, 47, 2> Version::_table_ = { offsetof(decltype(_table_), field_names), // no aux_entries &_Version_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::compiler::Version>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional string suffix = 4; {::_pbi::TcParser::FastSS1, @@ -752,6 +755,9 @@ const ::_pbi::TcParseTable<3, 5, 3, 79, 2> CodeGeneratorRequest::_table_ = { offsetof(decltype(_table_), aux_entries), &_CodeGeneratorRequest_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorRequest>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // repeated string file_to_generate = 1; @@ -1111,6 +1117,9 @@ const ::_pbi::TcParseTable<2, 4, 1, 86, 2> CodeGeneratorResponse_File::_table_ = offsetof(decltype(_table_), aux_entries), &_CodeGeneratorResponse_File_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorResponse_File>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional .google.protobuf.GeneratedCodeInfo generated_code_info = 16; {::_pbi::TcParser::FastMtS2, @@ -1411,6 +1420,9 @@ const ::_pbi::TcParseTable<2, 3, 1, 60, 2> CodeGeneratorResponse::_table_ = { offsetof(decltype(_table_), aux_entries), &_CodeGeneratorResponse_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorResponse>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string error = 1; diff --git a/src/google/protobuf/cpp_features.pb.cc b/src/google/protobuf/cpp_features.pb.cc index e4f9fc81f9..6664f4c7a3 100644 --- a/src/google/protobuf/cpp_features.pb.cc +++ b/src/google/protobuf/cpp_features.pb.cc @@ -201,6 +201,9 @@ const ::_pbi::TcParseTable<0, 1, 0, 0, 2> CppFeatures::_table_ = { offsetof(decltype(_table_), field_names), // no aux_entries &_CppFeatures_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::pb::CppFeatures>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { {::_pbi::TcParser::SingularVarintNoZag1(), diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 0a33adfd84..f3b563a9ab 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -2438,6 +2438,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> FileDescriptorSet::_table_ = { offsetof(decltype(_table_), aux_entries), &_FileDescriptorSet_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FileDescriptorSet>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // repeated .google.protobuf.FileDescriptorProto file = 1; {::_pbi::TcParser::FastMtR1, @@ -2718,6 +2721,9 @@ constexpr ::_pbi::TcParseTable<4, 13, 7, 79, 2> FileDescriptorProto::_table_ = { offsetof(decltype(_table_), aux_entries), &_FileDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FileDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -3281,6 +3287,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 1, 0, 2> DescriptorProto_ExtensionRange::_t offsetof(decltype(_table_), aux_entries), &_DescriptorProto_ExtensionRange_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto_ExtensionRange>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional int32 start = 1; @@ -3541,6 +3550,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 0, 2> DescriptorProto_ReservedRange::_ta offsetof(decltype(_table_), field_names), // no aux_entries &_DescriptorProto_ReservedRange_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto_ReservedRange>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional int32 end = 2; {::_pbi::TcParser::SingularVarintNoZag1<::uint32_t, offsetof(DescriptorProto_ReservedRange, _impl_.end_), 1>(), @@ -3815,6 +3827,9 @@ constexpr ::_pbi::TcParseTable<4, 10, 8, 65, 2> DescriptorProto::_table_ = { offsetof(decltype(_table_), aux_entries), &_DescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -4305,6 +4320,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 0, 71, 2> ExtensionRangeOptions_Declaration offsetof(decltype(_table_), field_names), // no aux_entries &_ExtensionRangeOptions_Declaration_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::ExtensionRangeOptions_Declaration>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional int32 number = 1; @@ -4639,6 +4657,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 4, 0, 12> ExtensionRangeOptions::_table_ = offsetof(decltype(_table_), aux_entries), &_ExtensionRangeOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::ExtensionRangeOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, {::_pbi::TcParser::MiniParse, {}}, @@ -5041,6 +5062,9 @@ constexpr ::_pbi::TcParseTable<4, 11, 3, 96, 2> FieldDescriptorProto::_table_ = offsetof(decltype(_table_), aux_entries), &_FieldDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FieldDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -5528,6 +5552,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 1, 49, 2> OneofDescriptorProto::_table_ = { offsetof(decltype(_table_), aux_entries), &_OneofDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::OneofDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional .google.protobuf.OneofOptions options = 2; {::_pbi::TcParser::FastMtS1, @@ -5766,6 +5793,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 0, 2> EnumDescriptorProto_EnumReservedRa offsetof(decltype(_table_), field_names), // no aux_entries &_EnumDescriptorProto_EnumReservedRange_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::EnumDescriptorProto_EnumReservedRange>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional int32 end = 2; {::_pbi::TcParser::SingularVarintNoZag1<::uint32_t, offsetof(EnumDescriptorProto_EnumReservedRange, _impl_.end_), 1>(), @@ -6025,6 +6055,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 3, 61, 2> EnumDescriptorProto::_table_ = { offsetof(decltype(_table_), aux_entries), &_EnumDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::EnumDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -6373,6 +6406,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 1, 53, 2> EnumValueDescriptorProto::_table_ offsetof(decltype(_table_), aux_entries), &_EnumValueDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::EnumValueDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -6667,6 +6703,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 2, 51, 2> ServiceDescriptorProto::_table_ = offsetof(decltype(_table_), aux_entries), &_ServiceDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::ServiceDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -6995,6 +7034,9 @@ constexpr ::_pbi::TcParseTable<3, 6, 1, 71, 2> MethodDescriptorProto::_table_ = offsetof(decltype(_table_), aux_entries), &_MethodDescriptorProto_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::MethodDescriptorProto>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string name = 1; @@ -7503,6 +7545,9 @@ constexpr ::_pbi::TcParseTable<5, 22, 3, 202, 12> FileOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_FileOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FileOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional string java_package = 1; @@ -8286,6 +8331,9 @@ constexpr ::_pbi::TcParseTable<3, 7, 2, 0, 7> MessageOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_MessageOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::MessageOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional bool message_set_wire_format = 1 [default = false]; @@ -8652,6 +8700,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 1, 57, 2> FieldOptions_EditionDefault::_tab offsetof(decltype(_table_), aux_entries), &_FieldOptions_EditionDefault_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FieldOptions_EditionDefault>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional string value = 2; {::_pbi::TcParser::FastSS1, @@ -8955,6 +9006,9 @@ constexpr ::_pbi::TcParseTable<4, 13, 7, 0, 7> FieldOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_FieldOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FieldOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional bool debug_redact = 16 [default = false]; {::_pbi::TcParser::FastV8S2, @@ -9478,6 +9532,9 @@ constexpr ::_pbi::TcParseTable<2, 2, 2, 0, 7> OneofOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_OneofOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::OneofOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional .google.protobuf.FeatureSet features = 1; @@ -9766,6 +9823,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 2, 0, 7> EnumOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_EnumOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::EnumOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, {::_pbi::TcParser::MiniParse, {}}, @@ -10124,6 +10184,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 2, 0, 7> EnumValueOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_EnumValueOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::EnumValueOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional bool deprecated = 1 [default = false]; @@ -10453,6 +10516,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 2, 0, 12> ServiceOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_ServiceOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::ServiceOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional bool deprecated = 33 [default = false]; @@ -10772,6 +10838,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 3, 0, 12> MethodOptions::_table_ = { offsetof(decltype(_table_), aux_entries), &_MethodOptions_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::MethodOptions>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional bool deprecated = 33 [default = false]; @@ -11088,6 +11157,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 62, 2> UninterpretedOption_NamePart::_ta offsetof(decltype(_table_), field_names), // no aux_entries &_UninterpretedOption_NamePart_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::UninterpretedOption_NamePart>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // required bool is_extension = 2; {::_pbi::TcParser::SingularVarintNoZag1(), @@ -11373,6 +11445,9 @@ constexpr ::_pbi::TcParseTable<3, 7, 1, 75, 2> UninterpretedOption::_table_ = { offsetof(decltype(_table_), aux_entries), &_UninterpretedOption_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::UninterpretedOption>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional string aggregate_value = 8; {::_pbi::TcParser::FastSS1, @@ -11758,6 +11833,9 @@ constexpr ::_pbi::TcParseTable<3, 6, 6, 0, 2> FeatureSet::_table_ = { offsetof(decltype(_table_), aux_entries), &_FeatureSet_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSet>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // optional .google.protobuf.FeatureSet.FieldPresence field_presence = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = { @@ -12112,6 +12190,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 2, 0, 2> FeatureSetDefaults_FeatureSetEditi offsetof(decltype(_table_), aux_entries), &_FeatureSetDefaults_FeatureSetEditionDefault_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSetDefaults_FeatureSetEditionDefault>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional .google.protobuf.FeatureSet features = 2; {::_pbi::TcParser::FastMtS1, @@ -12373,6 +12454,9 @@ constexpr ::_pbi::TcParseTable<1, 3, 3, 0, 2> FeatureSetDefaults::_table_ = { offsetof(decltype(_table_), aux_entries), &_FeatureSetDefaults_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSetDefaults>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // optional .google.protobuf.Edition minimum_edition = 4; {::_pbi::TcParser::FastEvS1, @@ -12652,6 +12736,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 0, 106, 2> SourceCodeInfo_Location::_table_ offsetof(decltype(_table_), field_names), // no aux_entries &_SourceCodeInfo_Location_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::SourceCodeInfo_Location>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // repeated int32 path = 1 [packed = true]; @@ -12960,6 +13047,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> SourceCodeInfo::_table_ = { offsetof(decltype(_table_), aux_entries), &_SourceCodeInfo_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::SourceCodeInfo>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // repeated .google.protobuf.SourceCodeInfo.Location location = 1; {::_pbi::TcParser::FastMtR1, @@ -13187,6 +13277,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 1, 64, 2> GeneratedCodeInfo_Annotation::_ta offsetof(decltype(_table_), aux_entries), &_GeneratedCodeInfo_Annotation_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::GeneratedCodeInfo_Annotation>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ {::_pbi::TcParser::MiniParse, {}}, // repeated int32 path = 1 [packed = true]; @@ -13490,6 +13583,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> GeneratedCodeInfo::_table_ = { offsetof(decltype(_table_), aux_entries), &_GeneratedCodeInfo_default_instance_._instance, ::_pbi::TcParser::GenericFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::GeneratedCodeInfo>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ // repeated .google.protobuf.GeneratedCodeInfo.Annotation annotation = 1; {::_pbi::TcParser::FastMtR1, diff --git a/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.txt b/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.txt index 77046ed34f..03be36fe32 100644 --- a/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.txt +++ b/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.txt @@ -1,7 +1,7 @@ [ RUN ] third_party/protobuf/editions/golden/simple_proto3.pb.cc @@ @@ - &_SimpleProto3_default_instance_._instance, - ::_pbi::TcParser::GenericFallbackLite, // fallback + ::_pbi::TcParser::GetTable<::protobuf_editions_test::golden::SimpleProto3>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE }, {{ - // optional int32 int32_field = 1; + // int32 int32_field = 1; diff --git a/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.xml b/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.xml index d2d397248f..4074bc2e17 100644 --- a/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.xml +++ b/src/google/protobuf/editions/golden/compare_cpp_codegen_failure.xml @@ -2,7 +2,7 @@ - + diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index a2e9c7cb83..d721060124 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -3224,6 +3224,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTableReflectionOnly() auto* full_table = ::new (p) Table{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, schema_.default_instance_, nullptr}, {{{&internal::TcParser::ReflectionParseLoop, {}}}}}; +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE + // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid + // branches. Here we don't know which field is the hottest, so set the pointer + // to itself to avoid nullptr. + full_table->header.to_prefetch = &full_table->header; +#endif // PROTOBUF_PREFETCH_PARSE_TABLE ABSL_DCHECK_EQ(static_cast(&full_table->header), static_cast(full_table)); return &full_table->header; @@ -3454,6 +3460,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { aux_offset, schema_.default_instance_, &internal::TcParser::ReflectionFallback}; +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE + // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid + // branches. Here we don't know which field is the hottest, so set the pointer + // to itself to avoid nullptr. + res->to_prefetch = res; +#endif // PROTOBUF_PREFETCH_PARSE_TABLE // Now copy the rest of the payloads PopulateTcParseFastEntries(table_info, res->fast_entry(0)); diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index 22aa54b681..55057a67fa 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -285,18 +285,30 @@ struct alignas(uint64_t) TcParseTableBase { // Handler for fields which are not handled by table dispatch. TailCallParseFunc fallback; + // A sub message's table to be prefetched. +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE + const TcParseTableBase* to_prefetch; +#endif // PROTOBUF_PREFETCH_PARSE_TABLE + // This constructor exactly follows the field layout, so it's technically // not necessary. However, it makes it much much easier to add or re-arrange // fields, because it can be overloaded with an additional constructor, // temporarily allowing both old and new protocol buffer headers to be // compiled. - constexpr TcParseTableBase( - uint16_t has_bits_offset, uint16_t extension_offset, - uint32_t max_field_number, uint8_t fast_idx_mask, - uint16_t lookup_table_offset, uint32_t skipmap32, - uint32_t field_entries_offset, uint16_t num_field_entries, - uint16_t num_aux_entries, uint32_t aux_offset, - const MessageLite* default_instance, TailCallParseFunc fallback) + constexpr TcParseTableBase(uint16_t has_bits_offset, + uint16_t extension_offset, + uint32_t max_field_number, uint8_t fast_idx_mask, + uint16_t lookup_table_offset, uint32_t skipmap32, + uint32_t field_entries_offset, + uint16_t num_field_entries, + uint16_t num_aux_entries, uint32_t aux_offset, + const MessageLite* default_instance, + TailCallParseFunc fallback +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE + , + const TcParseTableBase* to_prefetch +#endif // PROTOBUF_PREFETCH_PARSE_TABLE + ) : has_bits_offset(has_bits_offset), extension_offset(extension_offset), max_field_number(max_field_number), @@ -308,7 +320,13 @@ struct alignas(uint64_t) TcParseTableBase { num_aux_entries(num_aux_entries), aux_offset(aux_offset), default_instance(default_instance), - fallback(fallback) {} + fallback(fallback) +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE + , + to_prefetch(to_prefetch) +#endif // PROTOBUF_PREFETCH_PARSE_TABLE + { + } // Table entry for fast-path tailcall dispatch handling. struct FastFieldEntry { diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 21f21bd57a..1d68bd271e 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -789,6 +789,12 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), #define PROTOBUF_BUILTIN_CONSTANT_P(x) false #endif + +// TODO: Enable the feature by default and remove this flag. +#ifdef PROTOBUF_PREFETCH_PARSE_TABLE +#error PROTOBUF_PREFETCH_PARSE_TABLE was previously defined +#endif + // ThreadSafeArenaz is turned off completely in opensource builds. // autoheader defines this in some circumstances diff --git a/src/google/protobuf/port_undef.inc b/src/google/protobuf/port_undef.inc index 49aeb64865..3f20dfb0ba 100644 --- a/src/google/protobuf/port_undef.inc +++ b/src/google/protobuf/port_undef.inc @@ -80,6 +80,7 @@ #undef PROTOBUF_TSAN_WRITE #undef PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION #undef PROTOBUF_BUILTIN_CONSTANT_P +#undef PROTOBUF_PREFETCH_PARSE_TABLE #undef PROTOBUF_TC_PARAM_DECL #undef PROTOBUF_DEBUG #undef PROTO2_IS_OSS