Internal change.

PiperOrigin-RevId: 580726494

src/google/protobuf/compiler/cpp/helpers.cc

@@ -1024,6 +1024,13 @@ bool IsPresentMessage(const Descriptor* descriptor, const Options& options) {
   return true;
 }
 
+const FieldDescriptor* FindHottestField(
+    const std::vector<const FieldDescriptor*>& fields, const Options& options) {
+  (void)fields;
+  (void)options;
+  return nullptr;
+}
+
 static bool HasRepeatedFields(const Descriptor* descriptor) {
   for (int i = 0; i < descriptor->field_count(); ++i) {
     if (descriptor->field(i)->label() == FieldDescriptor::LABEL_REPEATED) {

src/google/protobuf/compiler/cpp/helpers.h

@@ -15,6 +15,7 @@
 #include <iterator>
 #include <string>
 #include <tuple>
+#include <vector>
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/log/absl_check.h"
@@ -429,6 +430,10 @@ bool ShouldForceAllocationOnConstruction(const Descriptor* desc,
 // Returns true if the message is present based on PDProto profile.
 bool IsPresentMessage(const Descriptor* descriptor, const Options& options);
 
+// Returns the most likely present field. Returns nullptr if not profile driven.
+const FieldDescriptor* FindHottestField(
+    const std::vector<const FieldDescriptor*>& fields, const Options& options);
+
 // Does the file contain any definitions that need extension_set.h?
 bool HasExtensionsOrExtendableMessage(const FileDescriptor* file);
 

src/google/protobuf/compiler/cpp/parse_function_generator.cc

@@ -401,9 +401,29 @@ void ParseFunctionGenerator::GenerateTailCallTable(io::Printer* printer) {
       }
       format(
           "&$1$._instance,\n"
-          "$2$,  // fallback\n"
+          "$2$,  // fallback\n",
-          "",
           DefaultInstanceName(descriptor_, options_), fallback);
+      std::vector<const FieldDescriptor*> subtable_fields;
+      for (const auto& aux : tc_table_info_->aux_entries) {
+        if (aux.type == internal::TailCallTableInfo::kSubTable) {
+          subtable_fields.push_back(aux.field);
+        }
+      }
+      const auto* hottest = FindHottestField(subtable_fields, options_);
+      // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
+      // branches. Set the pointer to itself to avoid nullptr.
+      printer->Emit(
+          {{"hottest_type_name",
+            QualifiedClassName(
+                hottest == nullptr ? descriptor_ : hottest->message_type(),
+                options_)}},
+          // clang-format off
+          R"cc(
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+::_pbi::TcParser::GetTable<$hottest_type_name$>(),  // to_prefetch
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
+          )cc");
+      // clang-format on
     }
     format("}, {{\n");
     {

src/google/protobuf/compiler/java/java_features.pb.cc

@@ -245,6 +245,9 @@ const ::_pbi::TcParseTable<1, 2, 1, 0, 2> JavaFeatures::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_JavaFeatures_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::pb::JavaFeatures>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional .pb.JavaFeatures.Utf8Validation utf8_validation = 2 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = {
     {::_pbi::TcParser::FastEr0S1,

src/google/protobuf/compiler/plugin.pb.cc

@@ -433,6 +433,9 @@ const ::_pbi::TcParseTable<2, 4, 0, 47, 2> Version::_table_ = {
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_Version_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::compiler::Version>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional string suffix = 4;
     {::_pbi::TcParser::FastSS1,
@@ -752,6 +755,9 @@ const ::_pbi::TcParseTable<3, 5, 3, 79, 2> CodeGeneratorRequest::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_CodeGeneratorRequest_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorRequest>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // repeated string file_to_generate = 1;
@@ -1111,6 +1117,9 @@ const ::_pbi::TcParseTable<2, 4, 1, 86, 2> CodeGeneratorResponse_File::_table_ =
     offsetof(decltype(_table_), aux_entries),
     &_CodeGeneratorResponse_File_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorResponse_File>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional .google.protobuf.GeneratedCodeInfo generated_code_info = 16;
     {::_pbi::TcParser::FastMtS2,
@@ -1411,6 +1420,9 @@ const ::_pbi::TcParseTable<2, 3, 1, 60, 2> CodeGeneratorResponse::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_CodeGeneratorResponse_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::compiler::CodeGeneratorResponse>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string error = 1;

src/google/protobuf/cpp_features.pb.cc

@@ -201,6 +201,9 @@ const ::_pbi::TcParseTable<0, 1, 0, 0, 2> CppFeatures::_table_ = {
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_CppFeatures_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::pb::CppFeatures>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional bool legacy_closed_enum = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = {
     {::_pbi::TcParser::SingularVarintNoZag1<bool, offsetof(CppFeatures, _impl_.legacy_closed_enum_), 0>(),

src/google/protobuf/descriptor.pb.cc

@@ -2438,6 +2438,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> FileDescriptorSet::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FileDescriptorSet_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FileDescriptorSet>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // repeated .google.protobuf.FileDescriptorProto file = 1;
     {::_pbi::TcParser::FastMtR1,
@@ -2718,6 +2721,9 @@ constexpr ::_pbi::TcParseTable<4, 13, 7, 79, 2> FileDescriptorProto::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FileDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FileDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -3281,6 +3287,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 1, 0, 2> DescriptorProto_ExtensionRange::_t
     offsetof(decltype(_table_), aux_entries),
     &_DescriptorProto_ExtensionRange_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto_ExtensionRange>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional int32 start = 1;
@@ -3541,6 +3550,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 0, 2> DescriptorProto_ReservedRange::_ta
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_DescriptorProto_ReservedRange_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto_ReservedRange>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional int32 end = 2;
     {::_pbi::TcParser::SingularVarintNoZag1<::uint32_t, offsetof(DescriptorProto_ReservedRange, _impl_.end_), 1>(),
@@ -3815,6 +3827,9 @@ constexpr ::_pbi::TcParseTable<4, 10, 8, 65, 2> DescriptorProto::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_DescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::DescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -4305,6 +4320,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 0, 71, 2> ExtensionRangeOptions_Declaration
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_ExtensionRangeOptions_Declaration_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::ExtensionRangeOptions_Declaration>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional int32 number = 1;
@@ -4639,6 +4657,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 4, 0, 12> ExtensionRangeOptions::_table_ =
     offsetof(decltype(_table_), aux_entries),
     &_ExtensionRangeOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::ExtensionRangeOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     {::_pbi::TcParser::MiniParse, {}},
@@ -5041,6 +5062,9 @@ constexpr ::_pbi::TcParseTable<4, 11, 3, 96, 2> FieldDescriptorProto::_table_ =
     offsetof(decltype(_table_), aux_entries),
     &_FieldDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FieldDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -5528,6 +5552,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 1, 49, 2> OneofDescriptorProto::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_OneofDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::OneofDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional .google.protobuf.OneofOptions options = 2;
     {::_pbi::TcParser::FastMtS1,
@@ -5766,6 +5793,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 0, 2> EnumDescriptorProto_EnumReservedRa
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_EnumDescriptorProto_EnumReservedRange_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::EnumDescriptorProto_EnumReservedRange>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional int32 end = 2;
     {::_pbi::TcParser::SingularVarintNoZag1<::uint32_t, offsetof(EnumDescriptorProto_EnumReservedRange, _impl_.end_), 1>(),
@@ -6025,6 +6055,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 3, 61, 2> EnumDescriptorProto::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_EnumDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::EnumDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -6373,6 +6406,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 1, 53, 2> EnumValueDescriptorProto::_table_
     offsetof(decltype(_table_), aux_entries),
     &_EnumValueDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::EnumValueDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -6667,6 +6703,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 2, 51, 2> ServiceDescriptorProto::_table_ =
     offsetof(decltype(_table_), aux_entries),
     &_ServiceDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::ServiceDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -6995,6 +7034,9 @@ constexpr ::_pbi::TcParseTable<3, 6, 1, 71, 2> MethodDescriptorProto::_table_ =
     offsetof(decltype(_table_), aux_entries),
     &_MethodDescriptorProto_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::MethodDescriptorProto>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string name = 1;
@@ -7503,6 +7545,9 @@ constexpr ::_pbi::TcParseTable<5, 22, 3, 202, 12> FileOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FileOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FileOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional string java_package = 1;
@@ -8286,6 +8331,9 @@ constexpr ::_pbi::TcParseTable<3, 7, 2, 0, 7> MessageOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_MessageOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::MessageOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional bool message_set_wire_format = 1 [default = false];
@@ -8652,6 +8700,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 1, 57, 2> FieldOptions_EditionDefault::_tab
     offsetof(decltype(_table_), aux_entries),
     &_FieldOptions_EditionDefault_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FieldOptions_EditionDefault>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional string value = 2;
     {::_pbi::TcParser::FastSS1,
@@ -8955,6 +9006,9 @@ constexpr ::_pbi::TcParseTable<4, 13, 7, 0, 7> FieldOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FieldOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FieldOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional bool debug_redact = 16 [default = false];
     {::_pbi::TcParser::FastV8S2,
@@ -9478,6 +9532,9 @@ constexpr ::_pbi::TcParseTable<2, 2, 2, 0, 7> OneofOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_OneofOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::OneofOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional .google.protobuf.FeatureSet features = 1;
@@ -9766,6 +9823,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 2, 0, 7> EnumOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_EnumOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::EnumOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     {::_pbi::TcParser::MiniParse, {}},
@@ -10124,6 +10184,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 2, 0, 7> EnumValueOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_EnumValueOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::EnumValueOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional bool deprecated = 1 [default = false];
@@ -10453,6 +10516,9 @@ constexpr ::_pbi::TcParseTable<2, 3, 2, 0, 12> ServiceOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_ServiceOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::ServiceOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional bool deprecated = 33 [default = false];
@@ -10772,6 +10838,9 @@ constexpr ::_pbi::TcParseTable<3, 4, 3, 0, 12> MethodOptions::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_MethodOptions_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::MethodOptions>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional bool deprecated = 33 [default = false];
@@ -11088,6 +11157,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 0, 62, 2> UninterpretedOption_NamePart::_ta
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_UninterpretedOption_NamePart_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::UninterpretedOption_NamePart>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // required bool is_extension = 2;
     {::_pbi::TcParser::SingularVarintNoZag1<bool, offsetof(UninterpretedOption_NamePart, _impl_.is_extension_), 1>(),
@@ -11373,6 +11445,9 @@ constexpr ::_pbi::TcParseTable<3, 7, 1, 75, 2> UninterpretedOption::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_UninterpretedOption_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::UninterpretedOption>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional string aggregate_value = 8;
     {::_pbi::TcParser::FastSS1,
@@ -11758,6 +11833,9 @@ constexpr ::_pbi::TcParseTable<3, 6, 6, 0, 2> FeatureSet::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FeatureSet_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSet>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // optional .google.protobuf.FeatureSet.FieldPresence field_presence = 1 [retention = RETENTION_RUNTIME, targets = TARGET_TYPE_FIELD, targets = TARGET_TYPE_FILE, edition_defaults = {
@@ -12112,6 +12190,9 @@ constexpr ::_pbi::TcParseTable<1, 2, 2, 0, 2> FeatureSetDefaults_FeatureSetEditi
     offsetof(decltype(_table_), aux_entries),
     &_FeatureSetDefaults_FeatureSetEditionDefault_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSetDefaults_FeatureSetEditionDefault>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional .google.protobuf.FeatureSet features = 2;
     {::_pbi::TcParser::FastMtS1,
@@ -12373,6 +12454,9 @@ constexpr ::_pbi::TcParseTable<1, 3, 3, 0, 2> FeatureSetDefaults::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_FeatureSetDefaults_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::FeatureSetDefaults>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // optional .google.protobuf.Edition minimum_edition = 4;
     {::_pbi::TcParser::FastEvS1,
@@ -12652,6 +12736,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 0, 106, 2> SourceCodeInfo_Location::_table_
     offsetof(decltype(_table_), field_names),  // no aux_entries
     &_SourceCodeInfo_Location_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::SourceCodeInfo_Location>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // repeated int32 path = 1 [packed = true];
@@ -12960,6 +13047,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> SourceCodeInfo::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_SourceCodeInfo_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::SourceCodeInfo>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // repeated .google.protobuf.SourceCodeInfo.Location location = 1;
     {::_pbi::TcParser::FastMtR1,
@@ -13187,6 +13277,9 @@ constexpr ::_pbi::TcParseTable<3, 5, 1, 64, 2> GeneratedCodeInfo_Annotation::_ta
     offsetof(decltype(_table_), aux_entries),
     &_GeneratedCodeInfo_Annotation_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::GeneratedCodeInfo_Annotation>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     {::_pbi::TcParser::MiniParse, {}},
     // repeated int32 path = 1 [packed = true];
@@ -13490,6 +13583,9 @@ constexpr ::_pbi::TcParseTable<0, 1, 1, 0, 2> GeneratedCodeInfo::_table_ = {
     offsetof(decltype(_table_), aux_entries),
     &_GeneratedCodeInfo_default_instance_._instance,
     ::_pbi::TcParser::GenericFallback,  // fallback
+    #ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+    ::_pbi::TcParser::GetTable<::google::protobuf::GeneratedCodeInfo>(),  // to_prefetch
+    #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   }, {{
     // repeated .google.protobuf.GeneratedCodeInfo.Annotation annotation = 1;
     {::_pbi::TcParser::FastMtR1,

src/google/protobuf/editions/golden/compare_cpp_codegen_failure.txt

@@ -1,7 +1,7 @@
 [ RUN      ] third_party/protobuf/editions/golden/simple_proto3.pb.cc
 @@ @@
-     &_SimpleProto3_default_instance_._instance,
+     ::_pbi::TcParser::GetTable<::protobuf_editions_test::golden::SimpleProto3>(),  // to_prefetch
-     ::_pbi::TcParser::GenericFallbackLite,  // fallback
+     #endif  // PROTOBUF_PREFETCH_PARSE_TABLE
    }, {{
 -    // optional int32 int32_field = 1;
 +    // int32 int32_field = 1;

src/google/protobuf/editions/golden/compare_cpp_codegen_failure.xml

@@ -2,7 +2,7 @@
 <testsuites tests="1" name="AllTests">
 <testsuite name="EditionsCodegenTests">
 <testcase name="third_party/protobuf/editions/golden/simple_proto3.pb.cc" status="run" result="completed" classname="DiffTest">
-<failure message="Value of: third_party/protobuf/editions/golden/simple_proto3.pb.cc&#x0A;Expected: &#x0A;// Generated by the protocol buffer compiler.  DO NOT EDIT!&#x0A;// source: third_party/protobuf/editions/golden/simple_proto3.proto&#x0A;&#x0A;#include &quot;third_party/protobuf/editions/golden/simple_proto3.pb.h&quot;&#x0A;&#x0A;#include &lt;algorithm&gt;&#x0A;#include &quot;third_party/protobuf/io/coded_stream.h&quot;&#x0A;#include &quot;third_party/protobuf/extension_set.h&quot;&#x0A;#include &quot;third_party/protobuf/wire_format_lite.h&quot;&#x0A;#include &quot;third_party/protobuf/io/zero_copy_stream_impl_lite.h&quot;&#x0A;#include &quot;third_party/protobuf/generated_message_tctable_impl.h&quot;&#x0A;// @@protoc_insertion_point(includes)&#x0A;&#x0A;// Must be included last.&#x0A;, with the difference:&#x0A;@@ @@&#x0A;     &amp;_SimpleProto3_default_instance_._instance,&#x0A;     ::_pbi::TcParser::GenericFallbackLite,  // fallback&#x0A;   }, {{&#x0A;-    // optional int32 int32_field = 1;&#x0A;+    // int32 int32_field = 1;&#x0A;     {::_pbi::TcParser::FastV32S1,&#x0A;      {8, 0, 0, PROTOBUF_FIELD_OFFSET(SimpleProto3, _impl_.int32_field_)}},&#x0A;   }}, {{&#x0A;     65535, 65535&#x0A;   }}, {{&#x0A;-    // optional int32 int32_field = 1;&#x0A;+    // int32 int32_field = 1;&#x0A;     {PROTOBUF_FIELD_OFFSET(SimpleProto3, _impl_.int32_field_), _Internal::kHasBitsOffset + 0, 0,&#x0A;     (0 | ::_fl::kFcOptional | ::_fl::kInt32)},&#x0A;   }},&#x0A;@@ @@&#x0A;   (void)cached_has_bits;&#x0A; &#x0A;   cached_has_bits = _impl_._has_bits_[0];&#x0A;-  // optional int32 int32_field = 1;&#x0A;+  // int32 int32_field = 1;&#x0A;   if (cached_has_bits &amp; 0x00000001u) {&#x0A;     target = ::proto2::internal::WireFormatLite::&#x0A;         WriteInt32ToArrayWithField&lt;1&gt;(&#x0A;@@ @@&#x0A;   // Prevent compiler warnings about cached_has_bits being unused&#x0A;   (void) cached_has_bits;&#x0A; &#x0A;-  // optional int32 int32_field = 1;&#x0A;+  // int32 int32_field = 1;&#x0A;   cached_has_bits = _impl_._has_bits_[0];&#x0A;   if (cached_has_bits &amp; 0x00000001u) {&#x0A;     total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(" type=""></failure>
+<failure message="Value of: third_party/protobuf/editions/golden/simple_proto3.pb.cc&#x0A;Expected: &#x0A;// Generated by the protocol buffer compiler.  DO NOT EDIT!&#x0A;// source: third_party/protobuf/editions/golden/simple_proto3.proto&#x0A;&#x0A;#include &quot;third_party/protobuf/editions/golden/simple_proto3.pb.h&quot;&#x0A;&#x0A;#include &lt;algorithm&gt;&#x0A;#include &quot;third_party/protobuf/io/coded_stream.h&quot;&#x0A;#include &quot;third_party/protobuf/extension_set.h&quot;&#x0A;#include &quot;third_party/protobuf/wire_format_lite.h&quot;&#x0A;#include &quot;third_party/protobuf/io/zero_copy_stream_impl_lite.h&quot;&#x0A;#include &quot;third_party/protobuf/generated_message_tctable_impl.h&quot;&#x0A;// @@protoc_insertion_point(includes)&#x0A;&#x0A;// Must be included last.&#x0A;, with the difference:&#x0A;@@ @@&#x0A;     ::_pbi::TcParser::GetTable&lt;::protobuf_editions_test::golden::SimpleProto3&gt;(),  // to_prefetch&#x0A;     #endif  // PROTOBUF_PREFETCH_PARSE_TABLE&#x0A;   }, {{&#x0A;-    // optional int32 int32_field = 1;&#x0A;+    // int32 int32_field = 1;&#x0A;     {::_pbi::TcParser::FastV32S1,&#x0A;      {8, 0, 0, PROTOBUF_FIELD_OFFSET(SimpleProto3, _impl_.int32_field_)}},&#x0A;   }}, {{&#x0A;     65535, 65535&#x0A;   }}, {{&#x0A;-    // optional int32 int32_field = 1;&#x0A;+    // int32 int32_field = 1;&#x0A;     {PROTOBUF_FIELD_OFFSET(SimpleProto3, _impl_.int32_field_), _Internal::kHasBitsOffset + 0, 0,&#x0A;     (0 | ::_fl::kFcOptional | ::_fl::kInt32)},&#x0A;   }},&#x0A;@@ @@&#x0A;   (void)cached_has_bits;&#x0A; &#x0A;   cached_has_bits = _impl_._has_bits_[0];&#x0A;-  // optional int32 int32_field = 1;&#x0A;+  // int32 int32_field = 1;&#x0A;   if (cached_has_bits &amp; 0x00000001u) {&#x0A;     target = ::proto2::internal::WireFormatLite::&#x0A;         WriteInt32ToArrayWithField&lt;1&gt;(&#x0A;@@ @@&#x0A;   // Prevent compiler warnings about cached_has_bits being unused&#x0A;   (void) cached_has_bits;&#x0A; &#x0A;-  // optional int32 int32_field = 1;&#x0A;+  // int32 int32_field = 1;&#x0A;   cached_has_bits = _impl_._has_bits_[0];&#x0A;   if (cached_has_bits &amp; 0x00000001u) {&#x0A;     total_size += ::_pbi::WireFormatLite::Int32SizePlusOne(" type=""></failure>
 </testcase>
 <testcase name="third_party/protobuf/editions/golden/simple_proto3.pb.h" status="run" result="completed" classname="DiffTest">
 <failure message="Value of: third_party/protobuf/editions/golden/simple_proto3.pb.h&#x0A;Expected: &#x0A;// Generated by the protocol buffer compiler.  DO NOT EDIT!&#x0A;// source: third_party/protobuf/editions/golden/simple_proto3.proto&#x0A;&#x0A;#ifndef GOOGLE_PROTOBUF_INCLUDED_third_5fparty_2fprotobuf_2feditions_2fgolden_2fsimple_5fproto3_2eproto_2epb_2eh&#x0A;#define GOOGLE_PROTOBUF_INCLUDED_third_5fparty_2fprotobuf_2feditions_2fgolden_2fsimple_5fproto3_2eproto_2epb_2eh&#x0A;&#x0A;#include &lt;limits&gt;&#x0A;#include &lt;string&gt;&#x0A;#include &lt;type_traits&gt;&#x0A;#include &lt;utility&gt;&#x0A;&#x0A;#include &quot;third_party/protobuf/io/coded_stream.h&quot;&#x0A;#include &quot;third_party/protobuf/arena.h&quot;&#x0A;#include &quot;third_party/protobuf/arenastring.h&quot;&#x0A;#include &quot;third_party/protobuf/generated_message_tctable_decl.h&quot;&#x0A;#include &quot;third_party/protobuf/generated_message_util.h&quot;&#x0A;#include &quot;third_party/protobuf/metadata_lite.h&quot;&#x0A;#include &quot;third_party/protobuf/message_lite.h&quot;&#x0A;// @@protoc_insertion_point(includes)&#x0A;&#x0A;// Must be included last.&#x0A;&#x0A;#endif  // GOOGLE_PROTOBUF_INCLUDED_third_5fparty_2fprotobuf_2feditions_2fgolden_2fsimple_5fproto3_2eproto_2epb_2eh&#x0A;, with the difference:&#x0A;@@ @@&#x0A;   enum : int {&#x0A;     kInt32FieldFieldNumber = 1,&#x0A;   };&#x0A;-  // optional int32 int32_field = 1;&#x0A;+  // int32 int32_field = 1;&#x0A;   bool has_int32_field() const;&#x0A;   void clear_int32_field() ;&#x0A;   ::int32_t int32_field() const;&#x0A;@@ @@&#x0A; &#x0A; // SimpleProto3&#x0A; &#x0A;-// optional int32 int32_field = 1;&#x0A;+// int32 int32_field = 1;&#x0A; inline bool SimpleProto3::has_int32_field() const {&#x0A;   bool value = (_impl_._has_bits_[0] &amp; 0x00000001u) != 0;&#x0A;   return value;" type=""></failure>

src/google/protobuf/generated_message_reflection.cc

@@ -3224,6 +3224,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTableReflectionOnly()
   auto* full_table = ::new (p)
       Table{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, schema_.default_instance_, nullptr},
             {{{&internal::TcParser::ReflectionParseLoop, {}}}}};
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+  // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
+  // branches. Here we don't know which field is the hottest, so set the pointer
+  // to itself to avoid nullptr.
+  full_table->header.to_prefetch = &full_table->header;
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
   ABSL_DCHECK_EQ(static_cast<void*>(&full_table->header),
                  static_cast<void*>(full_table));
   return &full_table->header;
@@ -3454,6 +3460,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
       aux_offset,
       schema_.default_instance_,
       &internal::TcParser::ReflectionFallback};
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+  // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
+  // branches. Here we don't know which field is the hottest, so set the pointer
+  // to itself to avoid nullptr.
+  res->to_prefetch = res;
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
 
   // Now copy the rest of the payloads
   PopulateTcParseFastEntries(table_info, res->fast_entry(0));

src/google/protobuf/generated_message_tctable_decl.h

@@ -285,18 +285,30 @@ struct alignas(uint64_t) TcParseTableBase {
   // Handler for fields which are not handled by table dispatch.
   TailCallParseFunc fallback;
 
+  // A sub message's table to be prefetched.
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+  const TcParseTableBase* to_prefetch;
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
+
   // This constructor exactly follows the field layout, so it's technically
   // not necessary.  However, it makes it much much easier to add or re-arrange
   // fields, because it can be overloaded with an additional constructor,
   // temporarily allowing both old and new protocol buffer headers to be
   // compiled.
-  constexpr TcParseTableBase(
+  constexpr TcParseTableBase(uint16_t has_bits_offset,
-      uint16_t has_bits_offset, uint16_t extension_offset,
+                             uint16_t extension_offset,
                              uint32_t max_field_number, uint8_t fast_idx_mask,
                              uint16_t lookup_table_offset, uint32_t skipmap32,
-      uint32_t field_entries_offset, uint16_t num_field_entries,
+                             uint32_t field_entries_offset,
+                             uint16_t num_field_entries,
                              uint16_t num_aux_entries, uint32_t aux_offset,
-      const MessageLite* default_instance, TailCallParseFunc fallback)
+                             const MessageLite* default_instance,
+                             TailCallParseFunc fallback
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+                             ,
+                             const TcParseTableBase* to_prefetch
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
+                             )
       : has_bits_offset(has_bits_offset),
         extension_offset(extension_offset),
         max_field_number(max_field_number),
@@ -308,7 +320,13 @@ struct alignas(uint64_t) TcParseTableBase {
         num_aux_entries(num_aux_entries),
         aux_offset(aux_offset),
         default_instance(default_instance),
-        fallback(fallback) {}
+        fallback(fallback)
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+        ,
+        to_prefetch(to_prefetch)
+#endif  // PROTOBUF_PREFETCH_PARSE_TABLE
+  {
+  }
 
   // Table entry for fast-path tailcall dispatch handling.
   struct FastFieldEntry {

src/google/protobuf/port_def.inc

@@ -789,6 +789,12 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3),
 #define PROTOBUF_BUILTIN_CONSTANT_P(x) false
 #endif
 
+
+// TODO: Enable the feature by default and remove this flag.
+#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
+#error PROTOBUF_PREFETCH_PARSE_TABLE was previously defined
+#endif
+
 // ThreadSafeArenaz is turned off completely in opensource builds.
 
 // autoheader defines this in some circumstances

src/google/protobuf/port_undef.inc

@@ -80,6 +80,7 @@
 #undef PROTOBUF_TSAN_WRITE
 #undef PROTOBUF_USE_TABLE_PARSER_ON_REFLECTION
 #undef PROTOBUF_BUILTIN_CONSTANT_P
+#undef PROTOBUF_PREFETCH_PARSE_TABLE
 #undef PROTOBUF_TC_PARAM_DECL
 #undef PROTOBUF_DEBUG
 #undef PROTO2_IS_OSS