Chiebot-Mirror
/
protobuf
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove hot-pathing between repeated and packed in fast path.

Browse Source 
PiperOrigin-RevId: 570099402
pull/14249/head
Protobuf Team Bot 1 year ago committed by Copybara-Service
parent 0a452fd359
commit 665006abd6
1 changed files with 20 additions and 89 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 93
      src/google/protobuf/generated_message_tctable_lite.cc

93
src/google/protobuf/generated_message_tctable_lite.cc
Unescape View File

@ -364,18 +364,6 @@ PROTOBUF_NOINLINE const char* TcParser::FastEndG2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return FastEndGroupImpl<uint16_t>(PROTOBUF_TC_PARAM_PASS);
}
namespace {
// InvertPacked changes tag bits from the given wire type to length
// delimited. This is the difference expected between packed and non-packed
// repeated fields.
template <WireFormatLite::WireType Wt>
inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) {
data.data ^= Wt ^ WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
}
} // namespace
//////////////////////////////////////////////////////////////////////////////
// Message fields
//////////////////////////////////////////////////////////////////////////////
@ -587,18 +575,8 @@ template <typename LayoutType, typename TagType>
PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedFixed(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
// Check if the field can be parsed as packed repeated:
constexpr WireFormatLite::WireType fallback_wt =
sizeof(LayoutType) == 4 ? WireFormatLite::WIRETYPE_FIXED32
: WireFormatLite::WIRETYPE_FIXED64;
InvertPacked<fallback_wt>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return PackedFixed<LayoutType, TagType>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
auto& field = RefAt<RepeatedField<LayoutType>>(msg, data.offset());
const auto tag = UnalignedLoad<TagType>(ptr);
do {
@ -628,26 +606,12 @@ PROTOBUF_NOINLINE const char* TcParser::FastF64R2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_TC_PARAM_PASS);
}
// Note: some versions of GCC will fail with error "function not inlinable" if
// corecursive functions are both marked with PROTOBUF_ALWAYS_INLINE (Clang
// accepts this). We can still apply the attribute to one of the two functions,
// just not both (so we do mark the Repeated variant as always inlined). This
// also applies to PackedVarint, below.
template <typename LayoutType, typename TagType>
const char* TcParser::PackedFixed(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::PackedFixed(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
// Try parsing as non-packed repeated:
constexpr WireFormatLite::WireType fallback_wt =
sizeof(LayoutType) == 4 ? WireFormatLite::WIRETYPE_FIXED32
: WireFormatLite::WIRETYPE_FIXED64;
InvertPacked<fallback_wt>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return RepeatedFixed<LayoutType, TagType>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
ptr += sizeof(TagType);
// Since ctx->ReadPackedFixed does not use TailCall<> or Return<>, sync any
// pending hasbits now:
@ -949,15 +913,8 @@ template <typename FieldType, typename TagType, bool zigzag>
PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedVarint(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
// Try parsing as non-packed repeated:
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return PackedVarint<FieldType, TagType, zigzag>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
auto& field = RefAt<RepeatedField<FieldType>>(msg, data.offset());
const auto expected_tag = UnalignedLoad<TagType>(ptr);
do {
@ -1017,18 +974,12 @@ PROTOBUF_NOINLINE const char* TcParser::FastZ64R2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_TC_PARAM_PASS);
}
// See comment on PackedFixed for why this is not PROTOBUF_ALWAYS_INLINE.
template <typename FieldType, typename TagType, bool zigzag>
const char* TcParser::PackedVarint(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::PackedVarint(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return RepeatedVarint<FieldType, TagType, zigzag>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
ptr += sizeof(TagType);
// Since ctx->ReadPackedVarint does not use TailCall or Return, sync any
// pending hasbits now:
@ -1168,16 +1119,11 @@ PROTOBUF_NOINLINE const char* TcParser::FastEvS2(PROTOBUF_TC_PARAM_DECL) {
}
template <typename TagType, uint16_t xform_val>
const char* TcParser::RepeatedEnum(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedEnum(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return PackedEnum<TagType, xform_val>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
auto& field = RefAt<RepeatedField<int32_t>>(msg, data.offset());
const auto expected_tag = UnalignedLoad<TagType>(ptr);
const TcParseTableBase::FieldAux aux = *table->field_aux(data.aux_idx());
@ -1225,16 +1171,11 @@ PROTOBUF_NOINLINE void TcParser::AddUnknownEnum(MessageLite* msg,
}
template <typename TagType, uint16_t xform_val>
const char* TcParser::PackedEnum(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::PackedEnum(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return RepeatedEnum<TagType, xform_val>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
const auto saved_tag = UnalignedLoad<TagType>(ptr);
ptr += sizeof(TagType);
// Since ctx->ReadPackedVarint does not use TailCall or Return, sync any
@ -1325,16 +1266,11 @@ PROTOBUF_NOINLINE const char* TcParser::FastEr1S2(PROTOBUF_TC_PARAM_DECL) {
}
template <typename TagType, uint8_t min>
const char* TcParser::RepeatedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedEnumSmallRange(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return PackedEnumSmallRange<TagType, min>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
auto& field = RefAt<RepeatedField<int32_t>>(msg, data.offset());
auto expected_tag = UnalignedLoad<TagType>(ptr);
const uint8_t max = data.aux_idx();
@ -1372,16 +1308,11 @@ PROTOBUF_NOINLINE const char* TcParser::FastEr1R2(PROTOBUF_TC_PARAM_DECL) {
}
template <typename TagType, uint8_t min>
const char* TcParser::PackedEnumSmallRange(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_ALWAYS_INLINE const char* TcParser::PackedEnumSmallRange(
PROTOBUF_TC_PARAM_DECL) {
if (PROTOBUF_PREDICT_FALSE(data.coded_tag<TagType>() != 0)) {
InvertPacked<WireFormatLite::WIRETYPE_VARINT>(data);
if (data.coded_tag<TagType>() == 0) {
PROTOBUF_MUSTTAIL return RepeatedEnumSmallRange<TagType, min>(
PROTOBUF_TC_PARAM_PASS);
} else {
PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_NO_DATA_PASS);
}
}
// Since ctx->ReadPackedVarint does not use TailCall or Return, sync any
// pending hasbits now:

Write Preview
Loading…
Cancel
Save