diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 1ba1b13922..14d0491396 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -72,6 +72,7 @@ class ParseFunctionGenerator::GeneratedOptionProvider final verify_flag(), IsStringInlined(field, gen_->options_), IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_), + /* use_direct_tcparser_table */ true, ShouldSplit(field, gen_->options_), }; } diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index bed6846ca8..3856082889 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -46,13 +46,11 @@ #include #include -#include "absl/log/absl_check.h" #include "google/protobuf/arenastring.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/extension_set.h" #include "google/protobuf/generated_message_reflection.h" -#include "google/protobuf/generated_message_tctable_decl.h" #include "google/protobuf/generated_message_util.h" #include "google/protobuf/map_field.h" #include "google/protobuf/map_field_inl.h" @@ -285,8 +283,6 @@ struct DynamicMessageFactory::TypeInfo { ~TypeInfo() { delete prototype; delete class_data.reflection; - ::operator delete( - const_cast(class_data.tc_table)); auto* type = class_data.descriptor; @@ -419,9 +415,9 @@ void DynamicMessage::SharedCtor(bool lock_factory) { new (field_ptr) Message*(nullptr); } else { if (IsMapFieldInApi(field)) { - // We need to lock in most cases to avoid data racing. - // When building the prototype via GetPrototypeNoLock, we construct - // this during crosslinking. + // We need to lock in most cases to avoid data racing. Only not lock + // when the constructor is called inside GetPrototype(), in which + // case we have already locked the factory. if (lock_factory) { if (arena != nullptr) { new (field_ptr) DynamicMapField( @@ -431,6 +427,17 @@ void DynamicMessage::SharedCtor(bool lock_factory) { new (field_ptr) DynamicMapField( type_info_->factory->GetPrototype(field->message_type())); } + } else { + if (arena != nullptr) { + new (field_ptr) + DynamicMapField(type_info_->factory->GetPrototypeNoLock( + field->message_type()), + arena); + } else { + new (field_ptr) + DynamicMapField(type_info_->factory->GetPrototypeNoLock( + field->message_type())); + } } } else { new (field_ptr) RepeatedPtrField(arena); @@ -576,18 +583,6 @@ void DynamicMessage::CrossLinkPrototypes() { factory->GetPrototypeNoLock(field->message_type()); } } - - // Construct the map fields. - // We need to delay this until we can do cross references. - for (int i = 0; i < descriptor->field_count(); i++) { - const FieldDescriptor* field = descriptor->field(i); - if (field->cpp_type() == field->CPPTYPE_MESSAGE && field->is_repeated() && - IsMapFieldInApi(field)) { - void* field_ptr = MutableRaw(i); - new (field_ptr) DynamicMapField( - type_info_->factory->GetPrototypeNoLock(field->message_type())); - } - } } Message* DynamicMessage::New(Arena* arena) const { @@ -779,16 +774,6 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock( type_info->class_data.reflection = new Reflection( type_info->class_data.descriptor, schema, type_info->pool, this); - type_info->class_data.reflection->CreateTcParseTable( - type_info->class_data.tc_table, [&](const auto* field) { - // The prototype we get might not necessarily be a dynamic message, so - // use GetClassData to fetch the table. - auto* table = - GetPrototypeNoLock(field->message_type())->GetTcParseTable(); - ABSL_DCHECK(table != nullptr); - return table; - }); - // Cross link prototypes. prototype->CrossLinkPrototypes(); diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 3419e5ac66..1443fb5ae1 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -25,7 +25,6 @@ #include "absl/base/call_once.h" #include "absl/base/const_init.h" #include "absl/container/flat_hash_set.h" -#include "absl/functional/function_ref.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/strings/str_format.h" @@ -3319,8 +3318,6 @@ void Reflection::PopulateTcParseEntries( void Reflection::PopulateTcParseFieldAux( const internal::TailCallTableInfo& table_info, - absl::FunctionRef - fetch_subtable, TcParseTableBase::FieldAux* field_aux) const { for (const auto& aux_entry : table_info.aux_entries) { switch (aux_entry.type) { @@ -3338,8 +3335,6 @@ void Reflection::PopulateTcParseFieldAux( field_aux++->offset = schema_.SizeofSplit(); break; case internal::TailCallTableInfo::kSubTable: - field_aux++->table = fetch_subtable(aux_entry.field); - break; case internal::TailCallTableInfo::kSubMessageWeak: case internal::TailCallTableInfo::kCreateInArena: case internal::TailCallTableInfo::kMessageVerifyFunc: @@ -3372,19 +3367,7 @@ void Reflection::PopulateTcParseFieldAux( } -const internal::TcParseTableBase* Reflection::GetTcParseTable() const { - absl::call_once(tcparse_table_once_, [&] { - CreateTcParseTable(tcparse_table_, [&](const auto* field) { - return GetDefaultMessageInstance(field)->GetTcParseTable(); - }); - }); - return tcparse_table_; -} - -void Reflection::CreateTcParseTable( - const TcParseTableBase*& out, - absl::FunctionRef - fetch_subtable) const { +const internal::TcParseTableBase* Reflection::CreateTcParseTable() const { using TcParseTableBase = internal::TcParseTableBase; std::vector fields; @@ -3428,6 +3411,11 @@ void Reflection::CreateTcParseTable( // Only LITE can be implicitly weak. /* is_implicitly_weak */ false, + + // We could change this to use direct table. + // Might be easier to do when all messages support TDP. + /* use_direct_tcparser_table */ false, + ref_.schema_.IsSplit(field), // }; } @@ -3484,12 +3472,6 @@ void Reflection::CreateTcParseTable( nullptr #endif // PROTOBUF_PREFETCH_PARSE_TABLE }; - - // Set the `out` pointer first. `fetch_subtable` is reentrant and we need to - // update the cache tables before the reentrancy. That way we can do cross - // referencing while building the tables. - out = res; - #ifdef PROTOBUF_PREFETCH_PARSE_TABLE // We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid // branches. Here we don't know which field is the hottest, so set the pointer @@ -3504,7 +3486,7 @@ void Reflection::CreateTcParseTable( PopulateTcParseEntries(table_info, res->field_entries_begin()); - PopulateTcParseFieldAux(table_info, fetch_subtable, res->field_aux(0u)); + PopulateTcParseFieldAux(table_info, res->field_aux(0u)); // Copy the name data. if (!table_info.field_name_data.empty()) { @@ -3515,6 +3497,8 @@ void Reflection::CreateTcParseTable( ABSL_CHECK_EQ(res->name_data() + table_info.field_name_data.size() - reinterpret_cast(res), byte_size); + + return res; } namespace { diff --git a/src/google/protobuf/generated_message_tctable_gen.cc b/src/google/protobuf/generated_message_tctable_gen.cc index e0dc36da7d..55bca3016b 100644 --- a/src/google/protobuf/generated_message_tctable_gen.cc +++ b/src/google/protobuf/generated_message_tctable_gen.cc @@ -204,12 +204,16 @@ TailCallTableInfo::FastFieldInfo::Field MakeFastFieldEntry( } break; case FieldDescriptor::TYPE_MESSAGE: - picked = HasLazyRep(field, options) - ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl) - : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt); + picked = + (HasLazyRep(field, options) ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl) + : options.use_direct_tcparser_table + ? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt) + : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMd)); break; case FieldDescriptor::TYPE_GROUP: - picked = PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt); + picked = (options.use_direct_tcparser_table + ? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt) + : PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGd)); break; } @@ -662,8 +666,10 @@ uint16_t MakeTypeCardForField( type_card |= 0 | fl::kMessage | fl::kRepGroup; if (options.is_implicitly_weak) { type_card |= fl::kTvWeakPtr; - } else { + } else if (options.use_direct_tcparser_table) { type_card |= fl::kTvTable; + } else { + type_card |= fl::kTvDefault; } break; case FieldDescriptor::TYPE_MESSAGE: @@ -678,8 +684,10 @@ uint16_t MakeTypeCardForField( } else { if (options.is_implicitly_weak) { type_card |= fl::kTvWeakPtr; - } else { + } else if (options.use_direct_tcparser_table) { type_card |= fl::kTvTable; + } else { + type_card |= fl::kTvDefault; } } } @@ -813,7 +821,7 @@ TailCallTableInfo::TailCallTableInfo( field->type() == FieldDescriptor::TYPE_GROUP) && !field->is_map() && !field->options().weak() && !HasLazyRep(field, options) && !options.is_implicitly_weak && - is_non_cold(options); + options.use_direct_tcparser_table && is_non_cold(options); }; for (const FieldDescriptor* field : ordered_fields) { if (is_non_cold_subtable(field)) { @@ -870,7 +878,9 @@ TailCallTableInfo::TailCallTableInfo( entry.aux_idx = TcParseTableBase::FieldEntry::kNoAuxIdx; } } else { - AuxType type = options.is_implicitly_weak ? kSubMessageWeak : kSubTable; + AuxType type = options.is_implicitly_weak ? kSubMessageWeak + : options.use_direct_tcparser_table ? kSubTable + : kSubMessage; if (message_options.should_profile_driven_cluster_aux_subtable && type == kSubTable && is_non_cold(options)) { aux_entries[subtable_aux_idx] = {type, {field}}; diff --git a/src/google/protobuf/generated_message_tctable_gen.h b/src/google/protobuf/generated_message_tctable_gen.h index 572bbd6bcb..667758519b 100644 --- a/src/google/protobuf/generated_message_tctable_gen.h +++ b/src/google/protobuf/generated_message_tctable_gen.h @@ -48,6 +48,7 @@ struct PROTOBUF_EXPORT TailCallTableInfo { field_layout::TransformValidation lazy_opt; bool is_string_inlined; bool is_implicitly_weak; + bool use_direct_tcparser_table; bool should_split; }; class OptionProvider { diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 10b08f65ac..838e748501 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -153,8 +153,9 @@ enum TransformValidation : uint16_t { kTvUtf8 = 2 << kTvShift, // proto3 // Message fields: - kTvTable = 1 << kTvShift, // Aux has TcParseTableBase* - kTvWeakPtr = 2 << kTvShift, // Aux has default_instance** (for weak) + kTvDefault = 1 << kTvShift, // Aux has default_instance* + kTvTable = 2 << kTvShift, // Aux has TcParseTableBase* + kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak) // Lazy message fields: kTvEager = 1 << kTvShift, @@ -350,7 +351,9 @@ inline void AlignFail(std::integral_constant, PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \ + PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \ PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \ @@ -569,14 +572,22 @@ class PROTOBUF_EXPORT TcParser final { // Functions referenced by generated fast tables (message types): // M: message G: group - // t: TcParseTable* (the contents of aux) l: lazy + // d: default* t: TcParseTable* (the contents of aux) l: lazy // S: singular R: repeated // 1/2: tag length (bytes) + PROTOBUF_NOINLINE static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL); + PROTOBUF_NOINLINE static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL); PROTOBUF_NOINLINE static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL); @@ -719,9 +730,9 @@ class PROTOBUF_EXPORT TcParser final { template static const char* MiniParse(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); template static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL); @@ -900,9 +911,6 @@ class PROTOBUF_EXPORT TcParser final { uint32_t field_num, ParseContext* ctx, MessageLite* msg); - static const TcParseTableBase* GetTableForAux(TcParseTableBase::FieldAux aux, - uint16_t type_card); - // UTF-8 validation: static void ReportFastUtf8Error(uint32_t decoded_tag, const TcParseTableBase* table); diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 35fb6ed556..f8b0d87942 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -351,7 +351,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastEndG2(PROTOBUF_TC_PARAM_DECL) { // Message fields ////////////////////////////////////////////////////////////////////////////// -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { @@ -363,32 +363,67 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( SyncHasbits(msg, hasbits, table); auto& field = RefAt(msg, data.offset()); - const auto* inner_table = table->field_aux(data.aux_idx())->table; - if (field == nullptr) { - field = inner_table->default_instance->New(msg->GetArena()); + if (aux_is_table) { + const auto* inner_table = table->field_aux(data.aux_idx())->table; + if (field == nullptr) { + field = inner_table->default_instance->New(msg->GetArena()); + } + const auto inner_loop = [&](const char* ptr) { + return ParseLoopInlined(field, ptr, ctx, inner_table); + }; + return group_coding ? ctx->ParseGroupInlined(ptr, FastDecodeTag(saved_tag), + inner_loop) + : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); + } else { + if (field == nullptr) { + const MessageLite* default_instance = + table->field_aux(data.aux_idx())->message_default(); + field = default_instance->New(msg->GetArena()); + } + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); + } + return ctx->ParseMessage(field, ptr); } - return group_coding ? ParseGroup(field, ptr, ctx, inner_table, - FastDecodeTag(saved_tag)) - : ParseMessage(field, ptr, ctx, inner_table); +} + +PROTOBUF_NOINLINE const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -406,7 +441,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastMlS2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_MUSTTAIL return LazyMessage(PROTOBUF_TC_PARAM_PASS); } -template +template inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { @@ -415,13 +450,25 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( const auto expected_tag = UnalignedLoad(ptr); const auto aux = *table->field_aux(data.aux_idx()); auto& field = RefAt(msg, data.offset()); - const MessageLite* const default_instance = aux.table->default_instance; + const MessageLite* const default_instance = + aux_is_table ? aux.table->default_instance : aux.message_default(); do { ptr += sizeof(TagType); MessageLite* submsg = field.AddMessage(default_instance); - ptr = group_coding ? ParseGroup(submsg, ptr, ctx, aux.table, - FastDecodeTag(expected_tag)) - : ParseMessage(submsg, ptr, ctx, aux.table); + if (aux_is_table) { + const auto inner_loop = [&](const char* ptr) { + return ParseLoopInlined(submsg, ptr, ctx, aux.table); + }; + ptr = group_coding ? ctx->ParseGroupInlined( + ptr, FastDecodeTag(expected_tag), inner_loop) + : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); + } else { + if (group_coding) { + ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag)); + } else { + ptr = ctx->ParseMessage(submsg, ptr); + } + } if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) { PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS); } @@ -433,23 +480,43 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); } +PROTOBUF_NOINLINE const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +PROTOBUF_NOINLINE const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + PROTOBUF_NOINLINE const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } PROTOBUF_NOINLINE const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -2224,16 +2291,6 @@ parse_loop: } -PROTOBUF_ALWAYS_INLINE inline const TcParseTableBase* TcParser::GetTableForAux( - TcParseTableBase::FieldAux aux, uint16_t type_card) { - if (ABSL_PREDICT_TRUE((type_card & field_layout::kTvMask) == - field_layout::kTvTable)) { - return aux.table; - } - ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, +field_layout::kTvWeakPtr); - return aux.message_default_weak()->GetTcParseTable(); -} - template PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { const auto& entry = RefAt(table, data.entry_offset()); @@ -2289,13 +2346,33 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { void* const base = MaybeGetSplitBase(msg, is_split, table); SyncHasbits(msg, hasbits, table); MessageLite*& field = RefAt(base, entry.offset); - - auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card); - if (need_init || field == nullptr) { - field = inner_table->default_instance->New(msg->GetArena()); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = table->field_aux(&entry)->table; + if (need_init || field == nullptr) { + field = inner_table->default_instance->New(msg->GetArena()); + } + const auto inner_loop = [&](const char* ptr) { + return ParseLoop(field, ptr, ctx, inner_table); + }; + return is_group ? ctx->ParseGroupInlined(ptr, decoded_tag, inner_loop) + : ctx->ParseLengthDelimitedInlined(ptr, inner_loop); + } else { + if (need_init || field == nullptr) { + const MessageLite* def; + if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) { + def = table->field_aux(&entry)->message_default(); + } else { + ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, + +field_layout::kTvWeakPtr); + def = table->field_aux(&entry)->message_default_weak(); + } + field = def->New(msg->GetArena()); + } + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag); + } + return ctx->ParseMessage(field, ptr); } - return is_group ? ParseGroup(field, ptr, ctx, inner_table, decoded_tag) - : ParseMessage(field, ptr, ctx, inner_table); } template @@ -2326,19 +2403,45 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) { RepeatedPtrFieldBase& field = MaybeCreateRepeatedRefAt( base, entry.offset, msg); - auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card); - const MessageLite* default_instance = inner_table->default_instance; - const char* ptr2 = ptr; - uint32_t next_tag; - do { - MessageLite* value = field.AddMessage(default_instance); - ptr = is_group ? ParseGroup(value, ptr2, ctx, inner_table, decoded_tag) - : ParseMessage(value, ptr2, ctx, inner_table); - if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; - if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; - ptr2 = ReadTag(ptr, &next_tag); - if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; - } while (next_tag == decoded_tag); + const auto aux = *table->field_aux(&entry); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = aux.table; + const MessageLite* default_instance = inner_table->default_instance; + const char* ptr2 = ptr; + uint32_t next_tag; + do { + MessageLite* value = field.AddMessage(default_instance); + const auto inner_loop = [&](const char* ptr) { + return ParseLoop(value, ptr, ctx, inner_table); + }; + ptr = is_group ? ctx->ParseGroupInlined(ptr2, decoded_tag, inner_loop) + : ctx->ParseLengthDelimitedInlined(ptr2, inner_loop); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; + ptr2 = ReadTag(ptr, &next_tag); + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; + } while (next_tag == decoded_tag); + } else { + const MessageLite* default_instance; + if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) { + default_instance = aux.message_default(); + } else { + ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, + +field_layout::kTvWeakPtr); + default_instance = aux.message_default_weak(); + } + const char* ptr2 = ptr; + uint32_t next_tag; + do { + MessageLite* value = field.AddMessage(default_instance); + ptr = is_group ? ctx->ParseGroup(value, ptr2, decoded_tag) + : ctx->ParseMessage(value, ptr2); + if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error; + if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop; + ptr2 = ReadTag(ptr, &next_tag); + if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error; + } while (next_tag == decoded_tag); + } PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS); parse_loop: PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS); @@ -2749,10 +2852,11 @@ std::string TypeCardToString(uint16_t type_card) { } static constexpr const char* kXFormNames[2][4] = { - {nullptr, "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}}; + {nullptr, "Default", "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}}; - static_assert((fl::kTvTable >> fl::kTvShift) == 1, ""); - static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 2, ""); + static_assert((fl::kTvDefault >> fl::kTvShift) == 1, ""); + static_assert((fl::kTvTable >> fl::kTvShift) == 2, ""); + static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 3, ""); static_assert((fl::kTvEager >> fl::kTvShift) == 1, ""); static_assert((fl::kTvLazy >> fl::kTvShift) == 2, ""); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 08a60ed782..fb0643c3d8 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1045,25 +1045,20 @@ class PROTOBUF_EXPORT Reflection final { using TcParseTableBase = internal::TcParseTableBase; mutable const TcParseTableBase* tcparse_table_ = nullptr; - const TcParseTableBase* GetTcParseTable() const; - - // We use an output parameter to ensure we write it as soon as we have the - // pointer. This allows DynamicMessageFactory to cross link as it is being - // built. - void CreateTcParseTable( - const TcParseTableBase*& out, - absl::FunctionRef - fetch_subtable) const; + const TcParseTableBase* GetTcParseTable() const { + absl::call_once(tcparse_table_once_, + [&] { tcparse_table_ = CreateTcParseTable(); }); + return tcparse_table_; + } + + const TcParseTableBase* CreateTcParseTable() const; void PopulateTcParseFastEntries( const internal::TailCallTableInfo& table_info, TcParseTableBase::FastFieldEntry* fast_entries) const; void PopulateTcParseEntries(internal::TailCallTableInfo& table_info, TcParseTableBase::FieldEntry* entries) const; - void PopulateTcParseFieldAux( - const internal::TailCallTableInfo& table_info, - absl::FunctionRef - fetch_subtable, - TcParseTableBase::FieldAux* field_aux) const; + void PopulateTcParseFieldAux(const internal::TailCallTableInfo& table_info, + TcParseTableBase::FieldAux* field_aux) const; template friend class RepeatedFieldRef; diff --git a/src/google/protobuf/message_lite.h b/src/google/protobuf/message_lite.h index 31c1ba9684..722bdf990e 100644 --- a/src/google/protobuf/message_lite.h +++ b/src/google/protobuf/message_lite.h @@ -53,7 +53,6 @@ class FastReflectionStringSetter; class Reflection; class Descriptor; class AssignDescriptorsHelper; -class DynamicMessageFactory; namespace io { @@ -663,7 +662,6 @@ class PROTOBUF_EXPORT MessageLite { friend class AssignDescriptorsHelper; friend class FastReflectionStringSetter; friend class Message; - friend class DynamicMessageFactory; friend class Reflection; friend class internal::DescriptorPoolExtensionFinder; friend class internal::ExtensionSet;