Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move all TDP parsing to use direct-to-table.

Browse Source 
DynamicMessage now eagerly creates the tc_table doing proper crosslinking of TDP tables.
Remove all kTvDefault cases from TcParser.

PiperOrigin-RevId: 620943275
pull/16344/head
Protobuf Team Bot 11 months ago committed by Copybara-Service
parent ab092fc810
commit 6a2bf46971
9 changed files with 138 additions and 224 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      src/google/protobuf/compiler/cpp/parse_function_generator.cc
  2. 43
      src/google/protobuf/dynamic_message.cc
  3. 34
      src/google/protobuf/generated_message_reflection.cc
  4. 26
      src/google/protobuf/generated_message_tctable_gen.cc
  5. 1
      src/google/protobuf/generated_message_tctable_gen.h
  6. 24
      src/google/protobuf/generated_message_tctable_impl.h
  7. 208
      src/google/protobuf/generated_message_tctable_lite.cc
  8. 23
      src/google/protobuf/message.h
  9. 2
      src/google/protobuf/message_lite.h

1
src/google/protobuf/compiler/cpp/parse_function_generator.cc
Unescape View File

@ -72,7 +72,6 @@ class ParseFunctionGenerator::GeneratedOptionProvider final
verify_flag(),
IsStringInlined(field, gen_->options_),
IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_),
/* use_direct_tcparser_table */ true,
ShouldSplit(field, gen_->options_),
};
}

43
src/google/protobuf/dynamic_message.cc
Unescape View File

@ -46,11 +46,13 @@
#include <memory>
#include <new>
#include "absl/log/absl_check.h"
#include "google/protobuf/arenastring.h"
#include "google/protobuf/descriptor.h"
#include "google/protobuf/descriptor.pb.h"
#include "google/protobuf/extension_set.h"
#include "google/protobuf/generated_message_reflection.h"
#include "google/protobuf/generated_message_tctable_decl.h"
#include "google/protobuf/generated_message_util.h"
#include "google/protobuf/map_field.h"
#include "google/protobuf/map_field_inl.h"
@ -283,6 +285,8 @@ struct DynamicMessageFactory::TypeInfo {
~TypeInfo() {
delete prototype;
delete class_data.reflection;
::operator delete(
const_cast<internal::TcParseTableBase*>(class_data.tc_table));
auto* type = class_data.descriptor;
@ -415,9 +419,9 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) Message*(nullptr);
} else {
if (IsMapFieldInApi(field)) {
// We need to lock in most cases to avoid data racing. Only not lock
// when the constructor is called inside GetPrototype(), in which
// case we have already locked the factory.
// We need to lock in most cases to avoid data racing.
// When building the prototype via GetPrototypeNoLock, we construct
// this during crosslinking.
if (lock_factory) {
if (arena != nullptr) {
new (field_ptr) DynamicMapField(
@ -427,17 +431,6 @@ void DynamicMessage::SharedCtor(bool lock_factory) {
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototype(field->message_type()));
}
} else {
if (arena != nullptr) {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()),
arena);
} else {
new (field_ptr)
DynamicMapField(type_info_->factory->GetPrototypeNoLock(
field->message_type()));
}
}
} else {
new (field_ptr) RepeatedPtrField<Message>(arena);
@ -583,6 +576,18 @@ void DynamicMessage::CrossLinkPrototypes() {
factory->GetPrototypeNoLock(field->message_type());
}
}
// Construct the map fields.
// We need to delay this until we can do cross references.
for (int i = 0; i < descriptor->field_count(); i++) {
const FieldDescriptor* field = descriptor->field(i);
if (field->cpp_type() == field->CPPTYPE_MESSAGE && field->is_repeated() &&
IsMapFieldInApi(field)) {
void* field_ptr = MutableRaw(i);
new (field_ptr) DynamicMapField(
type_info_->factory->GetPrototypeNoLock(field->message_type()));
}
}
}
Message* DynamicMessage::New(Arena* arena) const {
@ -774,6 +779,16 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
type_info->class_data.reflection = new Reflection(
type_info->class_data.descriptor, schema, type_info->pool, this);
type_info->class_data.reflection->CreateTcParseTable(
type_info->class_data.tc_table, [&](const auto* field) {
// The prototype we get might not necessarily be a dynamic message, so
// use GetClassData to fetch the table.
auto* table =
GetPrototypeNoLock(field->message_type())->GetTcParseTable();
ABSL_DCHECK(table != nullptr);
return table;
});
// Cross link prototypes.
prototype->CrossLinkPrototypes();

34
src/google/protobuf/generated_message_reflection.cc
Unescape View File

@ -25,6 +25,7 @@
#include "absl/base/call_once.h"
#include "absl/base/const_init.h"
#include "absl/container/flat_hash_set.h"
#include "absl/functional/function_ref.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h"
@ -3318,6 +3319,8 @@ void Reflection::PopulateTcParseEntries(
void Reflection::PopulateTcParseFieldAux(
const internal::TailCallTableInfo& table_info,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable,
TcParseTableBase::FieldAux* field_aux) const {
for (const auto& aux_entry : table_info.aux_entries) {
switch (aux_entry.type) {
@ -3335,6 +3338,8 @@ void Reflection::PopulateTcParseFieldAux(
field_aux++->offset = schema_.SizeofSplit();
break;
case internal::TailCallTableInfo::kSubTable:
field_aux++->table = fetch_subtable(aux_entry.field);
break;
case internal::TailCallTableInfo::kSubMessageWeak:
case internal::TailCallTableInfo::kCreateInArena:
case internal::TailCallTableInfo::kMessageVerifyFunc:
@ -3367,7 +3372,19 @@ void Reflection::PopulateTcParseFieldAux(
}
const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
const internal::TcParseTableBase* Reflection::GetTcParseTable() const {
absl::call_once(tcparse_table_once_, [&] {
CreateTcParseTable(tcparse_table_, [&](const auto* field) {
return GetDefaultMessageInstance(field)->GetTcParseTable();
});
});
return tcparse_table_;
}
void Reflection::CreateTcParseTable(
const TcParseTableBase*& out,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable) const {
using TcParseTableBase = internal::TcParseTableBase;
std::vector<const FieldDescriptor*> fields;
@ -3411,11 +3428,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
// Only LITE can be implicitly weak.
/* is_implicitly_weak */ false,
// We could change this to use direct table.
// Might be easier to do when all messages support TDP.
/* use_direct_tcparser_table */ false,
ref_.schema_.IsSplit(field), //
};
}
@ -3472,6 +3484,12 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
nullptr
#endif // PROTOBUF_PREFETCH_PARSE_TABLE
};
// Set the `out` pointer first. `fetch_subtable` is reentrant and we need to
// update the cache tables before the reentrancy. That way we can do cross
// referencing while building the tables.
out = res;
#ifdef PROTOBUF_PREFETCH_PARSE_TABLE
// We'll prefetch `to_prefetch->to_prefetch` unconditionally to avoid
// branches. Here we don't know which field is the hottest, so set the pointer
@ -3486,7 +3504,7 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
PopulateTcParseEntries(table_info, res->field_entries_begin());
PopulateTcParseFieldAux(table_info, res->field_aux(0u));
PopulateTcParseFieldAux(table_info, fetch_subtable, res->field_aux(0u));
// Copy the name data.
if (!table_info.field_name_data.empty()) {
@ -3497,8 +3515,6 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
ABSL_CHECK_EQ(res->name_data() + table_info.field_name_data.size() -
reinterpret_cast<char*>(res),
byte_size);
return res;
}
namespace {

26
src/google/protobuf/generated_message_tctable_gen.cc
Unescape View File

@ -204,16 +204,12 @@ TailCallTableInfo::FastFieldInfo::Field MakeFastFieldEntry(
}
break;
case FieldDescriptor::TYPE_MESSAGE:
picked =
(HasLazyRep(field, options) ? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMd));
picked = HasLazyRep(field, options)
? PROTOBUF_PICK_SINGLE_FUNCTION(kFastMl)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastMt);
break;
case FieldDescriptor::TYPE_GROUP:
picked = (options.use_direct_tcparser_table
? PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt)
: PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGd));
picked = PROTOBUF_PICK_REPEATABLE_FUNCTION(kFastGt);
break;
}
@ -666,10 +662,8 @@ uint16_t MakeTypeCardForField(
type_card |= 0 | fl::kMessage | fl::kRepGroup;
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
break;
case FieldDescriptor::TYPE_MESSAGE:
@ -684,10 +678,8 @@ uint16_t MakeTypeCardForField(
} else {
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
type_card |= fl::kTvTable;
} else {
type_card |= fl::kTvDefault;
type_card |= fl::kTvTable;
}
}
}
@ -821,7 +813,7 @@ TailCallTableInfo::TailCallTableInfo(
field->type() == FieldDescriptor::TYPE_GROUP) &&
!field->is_map() && !field->options().weak() &&
!HasLazyRep(field, options) && !options.is_implicitly_weak &&
options.use_direct_tcparser_table && is_non_cold(options);
is_non_cold(options);
};
for (const FieldDescriptor* field : ordered_fields) {
if (is_non_cold_subtable(field)) {
@ -878,9 +870,7 @@ TailCallTableInfo::TailCallTableInfo(
entry.aux_idx = TcParseTableBase::FieldEntry::kNoAuxIdx;
}
} else {
AuxType type = options.is_implicitly_weak ? kSubMessageWeak
: options.use_direct_tcparser_table ? kSubTable
: kSubMessage;
AuxType type = options.is_implicitly_weak ? kSubMessageWeak : kSubTable;
if (message_options.should_profile_driven_cluster_aux_subtable &&
type == kSubTable && is_non_cold(options)) {
aux_entries[subtable_aux_idx] = {type, {field}};

1
src/google/protobuf/generated_message_tctable_gen.h
Unescape View File

@ -48,7 +48,6 @@ struct PROTOBUF_EXPORT TailCallTableInfo {
field_layout::TransformValidation lazy_opt;
bool is_string_inlined;
bool is_implicitly_weak;
bool use_direct_tcparser_table;
bool should_split;
};
class OptionProvider {

24
src/google/protobuf/generated_message_tctable_impl.h
Unescape View File

@ -153,9 +153,8 @@ enum TransformValidation : uint16_t {
kTvUtf8 = 2 << kTvShift, // proto3
// Message fields:
kTvDefault = 1 << kTvShift, // Aux has default_instance*
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak)
kTvTable = 1 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 2 << kTvShift, // Aux has default_instance** (for weak)
// Lazy message fields:
kTvEager = 1 << kTvShift,
@ -351,9 +350,7 @@ inline void AlignFail(std::integral_constant<size_t, 1>,
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastBc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastSc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastUc) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastGt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMd) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_REPEATED(FastMt) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_SINGLE(FastMl) \
PROTOBUF_TC_PARSE_FUNCTION_LIST_END_GROUP() \
@ -572,22 +569,14 @@ class PROTOBUF_EXPORT TcParser final {
// Functions referenced by generated fast tables (message types):
// M: message G: group
// d: default* t: TcParseTable* (the contents of aux) l: lazy
// t: TcParseTable* (the contents of aux) l: lazy
// S: singular R: repeated
// 1/2: tag length (bytes)
PROTOBUF_NOINLINE static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL);
PROTOBUF_NOINLINE static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL);
@ -730,9 +719,9 @@ class PROTOBUF_EXPORT TcParser final {
template <bool export_called_function>
static const char* MiniParse(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType>
static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL);
@ -911,6 +900,9 @@ class PROTOBUF_EXPORT TcParser final {
uint32_t field_num, ParseContext* ctx,
MessageLite* msg);
static const TcParseTableBase* GetTableForAux(TcParseTableBase::FieldAux aux,
uint16_t type_card);
// UTF-8 validation:
static void ReportFastUtf8Error(uint32_t decoded_tag,
const TcParseTableBase* table);

208
src/google/protobuf/generated_message_tctable_lite.cc
Unescape View File

@ -351,7 +351,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastEndG2(PROTOBUF_TC_PARAM_DECL) {
// Message fields
//////////////////////////////////////////////////////////////////////////////
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
@ -363,67 +363,32 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl(
SyncHasbits(msg, hasbits, table);
auto& field = RefAt<MessageLite*>(msg, data.offset());
if (aux_is_table) {
const auto* inner_table = table->field_aux(data.aux_idx())->table;
if (field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
const auto inner_loop = [&](const char* ptr) {
return ParseLoopInlined(field, ptr, ctx, inner_table);
};
return group_coding ? ctx->ParseGroupInlined(ptr, FastDecodeTag(saved_tag),
inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (field == nullptr) {
const MessageLite* default_instance =
table->field_aux(data.aux_idx())->message_default();
field = default_instance->New(msg->GetArena());
}
if (group_coding) {
return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag));
}
return ctx->ParseMessage(field, ptr);
const auto* inner_table = table->field_aux(data.aux_idx())->table;
if (field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
}
PROTOBUF_NOINLINE const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
return group_coding ? ParseGroup(field, ptr, ctx, inner_table,
FastDecodeTag(saved_tag))
: ParseMessage(field, ptr, ctx, inner_table);
}
PROTOBUF_NOINLINE const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false, true>(
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false, true>(
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true, true>(
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint8_t, true>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true, true>(
PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl<uint16_t, true>(
PROTOBUF_TC_PARAM_PASS);
}
@ -441,7 +406,7 @@ PROTOBUF_NOINLINE const char* TcParser::FastMlS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return LazyMessage<uint16_t>(PROTOBUF_TC_PARAM_PASS);
}
template <typename TagType, bool group_coding, bool aux_is_table>
template <typename TagType, bool group_coding>
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
@ -450,25 +415,13 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
const auto expected_tag = UnalignedLoad<TagType>(ptr);
const auto aux = *table->field_aux(data.aux_idx());
auto& field = RefAt<RepeatedPtrFieldBase>(msg, data.offset());
const MessageLite* const default_instance =
aux_is_table ? aux.table->default_instance : aux.message_default();
const MessageLite* const default_instance = aux.table->default_instance;
do {
ptr += sizeof(TagType);
MessageLite* submsg = field.AddMessage(default_instance);
if (aux_is_table) {
const auto inner_loop = [&](const char* ptr) {
return ParseLoopInlined(submsg, ptr, ctx, aux.table);
};
ptr = group_coding ? ctx->ParseGroupInlined(
ptr, FastDecodeTag(expected_tag), inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (group_coding) {
ptr = ctx->ParseGroup(submsg, ptr, FastDecodeTag(expected_tag));
} else {
ptr = ctx->ParseMessage(submsg, ptr);
}
}
ptr = group_coding ? ParseGroup(submsg, ptr, ctx, aux.table,
FastDecodeTag(expected_tag))
: ParseMessage(submsg, ptr, ctx, aux.table);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) {
PROTOBUF_MUSTTAIL return Error(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
@ -480,43 +433,23 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false, true>(
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false, true>(
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, false>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true, true>(
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint8_t, true>(
PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true, true>(
PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl<uint16_t, true>(
PROTOBUF_TC_PARAM_PASS);
}
@ -2291,6 +2224,16 @@ parse_loop:
}
PROTOBUF_ALWAYS_INLINE inline const TcParseTableBase* TcParser::GetTableForAux(
TcParseTableBase::FieldAux aux, uint16_t type_card) {
if (ABSL_PREDICT_TRUE((type_card & field_layout::kTvMask) ==
field_layout::kTvTable)) {
return aux.table;
}
ABSL_DCHECK_EQ(type_card & field_layout::kTvMask, +field_layout::kTvWeakPtr);
return aux.message_default_weak()->GetTcParseTable();
}
template <bool is_split>
PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
const auto& entry = RefAt<FieldEntry>(table, data.entry_offset());
@ -2346,33 +2289,13 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
void* const base = MaybeGetSplitBase(msg, is_split, table);
SyncHasbits(msg, hasbits, table);
MessageLite*& field = RefAt<MessageLite*>(base, entry.offset);
if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) {
auto* inner_table = table->field_aux(&entry)->table;
if (need_init || field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
const auto inner_loop = [&](const char* ptr) {
return ParseLoop(field, ptr, ctx, inner_table);
};
return is_group ? ctx->ParseGroupInlined(ptr, decoded_tag, inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr, inner_loop);
} else {
if (need_init || field == nullptr) {
const MessageLite* def;
if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) {
def = table->field_aux(&entry)->message_default();
} else {
ABSL_DCHECK_EQ(type_card & field_layout::kTvMask,
+field_layout::kTvWeakPtr);
def = table->field_aux(&entry)->message_default_weak();
}
field = def->New(msg->GetArena());
}
if (is_group) {
return ctx->ParseGroup(field, ptr, decoded_tag);
}
return ctx->ParseMessage(field, ptr);
auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card);
if (need_init || field == nullptr) {
field = inner_table->default_instance->New(msg->GetArena());
}
return is_group ? ParseGroup(field, ptr, ctx, inner_table, decoded_tag)
: ParseMessage(field, ptr, ctx, inner_table);
}
template <bool is_split, bool is_group>
@ -2403,45 +2326,19 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) {
RepeatedPtrFieldBase& field =
MaybeCreateRepeatedRefAt<RepeatedPtrFieldBase, is_split>(
base, entry.offset, msg);
const auto aux = *table->field_aux(&entry);
if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) {
auto* inner_table = aux.table;
const MessageLite* default_instance = inner_table->default_instance;
const char* ptr2 = ptr;
uint32_t next_tag;
do {
MessageLite* value = field.AddMessage(default_instance);
const auto inner_loop = [&](const char* ptr) {
return ParseLoop(value, ptr, ctx, inner_table);
};
ptr = is_group ? ctx->ParseGroupInlined(ptr2, decoded_tag, inner_loop)
: ctx->ParseLengthDelimitedInlined(ptr2, inner_loop);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
} while (next_tag == decoded_tag);
} else {
const MessageLite* default_instance;
if ((type_card & field_layout::kTvMask) == field_layout::kTvDefault) {
default_instance = aux.message_default();
} else {
ABSL_DCHECK_EQ(type_card & field_layout::kTvMask,
+field_layout::kTvWeakPtr);
default_instance = aux.message_default_weak();
}
const char* ptr2 = ptr;
uint32_t next_tag;
do {
MessageLite* value = field.AddMessage(default_instance);
ptr = is_group ? ctx->ParseGroup(value, ptr2, decoded_tag)
: ctx->ParseMessage(value, ptr2);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
} while (next_tag == decoded_tag);
}
auto* inner_table = GetTableForAux(*table->field_aux(&entry), type_card);
const MessageLite* default_instance = inner_table->default_instance;
const char* ptr2 = ptr;
uint32_t next_tag;
do {
MessageLite* value = field.AddMessage(default_instance);
ptr = is_group ? ParseGroup(value, ptr2, ctx, inner_table, decoded_tag)
: ParseMessage(value, ptr2, ctx, inner_table);
if (PROTOBUF_PREDICT_FALSE(ptr == nullptr)) goto error;
if (PROTOBUF_PREDICT_FALSE(!ctx->DataAvailable(ptr))) goto parse_loop;
ptr2 = ReadTag(ptr, &next_tag);
if (PROTOBUF_PREDICT_FALSE(ptr2 == nullptr)) goto error;
} while (next_tag == decoded_tag);
PROTOBUF_MUSTTAIL return ToTagDispatch(PROTOBUF_TC_PARAM_NO_DATA_PASS);
parse_loop:
PROTOBUF_MUSTTAIL return ToParseLoop(PROTOBUF_TC_PARAM_NO_DATA_PASS);
@ -2852,11 +2749,10 @@ std::string TypeCardToString(uint16_t type_card) {
}
static constexpr const char* kXFormNames[2][4] = {
{nullptr, "Default", "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}};
{nullptr, "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}};
static_assert((fl::kTvDefault >> fl::kTvShift) == 1, "");
static_assert((fl::kTvTable >> fl::kTvShift) == 2, "");
static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 3, "");
static_assert((fl::kTvTable >> fl::kTvShift) == 1, "");
static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 2, "");
static_assert((fl::kTvEager >> fl::kTvShift) == 1, "");
static_assert((fl::kTvLazy >> fl::kTvShift) == 2, "");

23
src/google/protobuf/message.h
Unescape View File

@ -1045,20 +1045,25 @@ class PROTOBUF_EXPORT Reflection final {
using TcParseTableBase = internal::TcParseTableBase;
mutable const TcParseTableBase* tcparse_table_ = nullptr;
const TcParseTableBase* GetTcParseTable() const {
absl::call_once(tcparse_table_once_,
[&] { tcparse_table_ = CreateTcParseTable(); });
return tcparse_table_;
}
const TcParseTableBase* CreateTcParseTable() const;
const TcParseTableBase* GetTcParseTable() const;
// We use an output parameter to ensure we write it as soon as we have the
// pointer. This allows DynamicMessageFactory to cross link as it is being
// built.
void CreateTcParseTable(
const TcParseTableBase*& out,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable) const;
void PopulateTcParseFastEntries(
const internal::TailCallTableInfo& table_info,
TcParseTableBase::FastFieldEntry* fast_entries) const;
void PopulateTcParseEntries(internal::TailCallTableInfo& table_info,
TcParseTableBase::FieldEntry* entries) const;
void PopulateTcParseFieldAux(const internal::TailCallTableInfo& table_info,
TcParseTableBase::FieldAux* field_aux) const;
void PopulateTcParseFieldAux(
const internal::TailCallTableInfo& table_info,
absl::FunctionRef<const TcParseTableBase*(const FieldDescriptor*)>
fetch_subtable,
TcParseTableBase::FieldAux* field_aux) const;
template <typename T, typename Enable>
friend class RepeatedFieldRef;

2
src/google/protobuf/message_lite.h
Unescape View File

@ -53,6 +53,7 @@ class FastReflectionStringSetter;
class Reflection;
class Descriptor;
class AssignDescriptorsHelper;
class DynamicMessageFactory;
namespace io {
@ -662,6 +663,7 @@ class PROTOBUF_EXPORT MessageLite {
friend class AssignDescriptorsHelper;
friend class FastReflectionStringSetter;
friend class Message;
friend class DynamicMessageFactory;
friend class Reflection;
friend class internal::DescriptorPoolExtensionFinder;
friend class internal::ExtensionSet;

Write Preview
Loading…
Cancel
Save