@ -47,6 +47,7 @@
# include "google/protobuf/message_lite.h"
# include "google/protobuf/parse_context.h"
# include "google/protobuf/repeated_field.h"
# include "google/protobuf/repeated_ptr_field.h"
# include "google/protobuf/varint_shuffle.h"
# include "google/protobuf/wire_format_lite.h"
# include "utf8_validity.h"
@ -300,22 +301,22 @@ inline PROTOBUF_ALWAYS_INLINE const char* TcParser::MiniParse(
entry - > type_card & ( + field_layout : : kSplitMask | FieldKind : : kFkMask ) ;
static constexpr TailCallParseFunc kMiniParseTable [ ] = {
& MpFallback , // FieldKind::kFkNone
& MpVarint < false > , // FieldKind::kFkVarint
& MpPackedVarint , // FieldKind::kFkPackedVarint
& MpFixed < false > , // FieldKind::kFkFixed
& MpPackedFixed , // FieldKind::kFkPackedFixed
& MpString < false > , // FieldKind::kFkString
& MpMessage < false > , // FieldKind::kFkMessage
& MpMap , // FieldKind::kFkMap
& Error , // kSplitMask | FieldKind::kFkNone
& MpVarint < true > , // kSplitMask | FieldKind::kFkVarint
& Error , // kSplitMask | FieldKind::kFkPackedVarint
& MpFixed < true > , // kSplitMask | FieldKind::kFkFixed
& Error , // kSplitMask | FieldKind::kFkPackedFixed
& MpString < true > , // kSplitMask | FieldKind::kFkString
& MpMessage < true > , // kSplitMask | FieldKind::kFkMessage
& Error , // kSplitMask | FieldKind::kFkMap
& MpFallback , // FieldKind::kFkNone
& MpVarint < false > , // FieldKind::kFkVarint
& MpPackedVarint < false > , // FieldKind::kFkPackedVarint
& MpFixed < false > , // FieldKind::kFkFixed
& MpPackedFixed < false > , // FieldKind::kFkPackedFixed
& MpString < false > , // FieldKind::kFkString
& MpMessage < false > , // FieldKind::kFkMessage
& MpMap < false > , // FieldKind::kFkMap
& Error , // kSplitMask | FieldKind::kFkNone
& MpVarint < true > , // kSplitMask | FieldKind::kFkVarint
& MpPackedVarint < true > , // kSplitMask | FieldKind::kFkPackedVarint
& MpFixed < true > , // kSplitMask | FieldKind::kFkFixed
& MpPackedFixed < true > , // kSplitMask | FieldKind::kFkPackedFixed
& MpString < true > , // kSplitMask | FieldKind::kFkString
& MpMessage < true > , // kSplitMask | FieldKind::kFkMessage
& MpMap < true > , // kSplitMask | FieldKind::kFkMap
} ;
// Just to be sure we got the order right, above.
static_assert ( 0 = = FieldKind : : kFkNone , " Invalid table order " ) ;
@ -1796,7 +1797,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) {
// Check for repeated parsing (wiretype fallback is handled there):
if ( card = = field_layout : : kFcRepeated ) {
PROTOBUF_MUSTTAIL return MpRepeatedFixed ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpRepeatedFixed < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
// Check for mismatched wiretype:
const uint16_t rep = type_card & field_layout : : kRepMask ;
@ -1829,6 +1830,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return ToTagDispatch ( PROTOBUF_TC_PARAM_NO_DATA_PASS ) ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpRepeatedFixed (
PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
@ -1837,16 +1839,18 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed(
// Check for packed repeated fallback:
if ( decoded_wiretype = = WireFormatLite : : WIRETYPE_LENGTH_DELIMITED ) {
PROTOBUF_MUSTTAIL return MpPackedFixed ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpPackedFixed < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
const uint16_t type_card = entry . type_card ;
const uint16_t rep = type_card & field_layout : : kRepMask ;
if ( rep = = field_layout : : kRep64Bits ) {
if ( decoded_wiretype ! = WireFormatLite : : WIRETYPE_FIXED64 ) {
PROTOBUF_MUSTTAIL return table - > fallback ( PROTOBUF_TC_PARAM_PASS ) ;
}
auto & field = RefAt < RepeatedField < uint64_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint64_t , is_split > (
base , entry . offset , msg ) ;
constexpr auto size = sizeof ( uint64_t ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
@ -1863,7 +1867,8 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedFixed(
if ( decoded_wiretype ! = WireFormatLite : : WIRETYPE_FIXED32 ) {
PROTOBUF_MUSTTAIL return table - > fallback ( PROTOBUF_TC_PARAM_PASS ) ;
}
auto & field = RefAt < RepeatedField < uint32_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint32_t , is_split > (
base , entry . offset , msg ) ;
constexpr auto size = sizeof ( uint32_t ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
@ -1884,6 +1889,7 @@ error:
PROTOBUF_MUSTTAIL return Error ( PROTOBUF_TC_PARAM_NO_DATA_PASS ) ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpPackedFixed ( PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
const uint16_t type_card = entry . type_card ;
@ -1891,17 +1897,20 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedFixed(PROTOBUF_TC_PARAM_DECL) {
// Check for non-packed repeated fallback:
if ( decoded_wiretype ! = WireFormatLite : : WIRETYPE_LENGTH_DELIMITED ) {
PROTOBUF_MUSTTAIL return MpRepeatedFixed ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpRepeatedFixed < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
int size = ReadSize ( & ptr ) ;
uint16_t rep = type_card & field_layout : : kRepMask ;
if ( rep = = field_layout : : kRep64Bits ) {
auto & field = RefAt < RepeatedField < uint64_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint64_t , is_split > (
base , entry . offset , msg ) ;
ptr = ctx - > ReadPackedFixed ( ptr , size , & field ) ;
} else {
ABSL_DCHECK_EQ ( rep , static_cast < uint16_t > ( field_layout : : kRep32Bits ) ) ;
auto & field = RefAt < RepeatedField < uint32_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint32_t , is_split > (
base , entry . offset , msg ) ;
ptr = ctx - > ReadPackedFixed ( ptr , size , & field ) ;
}
@ -1919,7 +1928,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) {
// Check for repeated parsing:
if ( card = = field_layout : : kFcRepeated ) {
PROTOBUF_MUSTTAIL return MpRepeatedVarint ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpRepeatedVarint < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
// Check for wire type mismatch:
if ( ( data . tag ( ) & 7 ) ! = WireFormatLite : : WIRETYPE_VARINT ) {
@ -1975,6 +1984,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return ToTagDispatch ( PROTOBUF_TC_PARAM_NO_DATA_PASS ) ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpRepeatedVarint (
PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
@ -1984,7 +1994,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
// Check for packed repeated fallback:
if ( decoded_wiretype = = WireFormatLite : : WIRETYPE_LENGTH_DELIMITED ) {
PROTOBUF_MUSTTAIL return MpPackedVarint ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpPackedVarint < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
// Check for wire type mismatch:
if ( decoded_wiretype ! = WireFormatLite : : WIRETYPE_VARINT ) {
@ -1994,9 +2004,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
const bool is_zigzag = xform_val = = field_layout : : kTvZigZag ;
const bool is_validated_enum = xform_val & field_layout : : kTvEnum ;
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
uint16_t rep = type_card & field_layout : : kRepMask ;
if ( rep = = field_layout : : kRep64Bits ) {
auto & field = RefAt < RepeatedField < uint64_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint64_t , is_split > (
base , entry . offset , msg ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
do {
@ -2009,7 +2021,8 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
if ( PROTOBUF_PREDICT_FALSE ( ptr2 = = nullptr ) ) goto error ;
} while ( next_tag = = decoded_tag ) ;
} else if ( rep = = field_layout : : kRep32Bits ) {
auto & field = RefAt < RepeatedField < uint32_t > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedFieldRefAt < uint32_t , is_split > (
base , entry . offset , msg ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
do {
@ -2032,7 +2045,8 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedVarint(
} while ( next_tag = = decoded_tag ) ;
} else {
ABSL_DCHECK_EQ ( rep , static_cast < uint16_t > ( field_layout : : kRep8Bits ) ) ;
auto & field = RefAt < RepeatedField < bool > > ( msg , entry . offset ) ;
auto & field =
MaybeCreateRepeatedFieldRefAt < bool , is_split > ( base , entry . offset , msg ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
do {
@ -2053,6 +2067,7 @@ error:
PROTOBUF_MUSTTAIL return Error ( PROTOBUF_TC_PARAM_NO_DATA_PASS ) ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpPackedVarint ( PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
auto type_card = entry . type_card ;
@ -2060,7 +2075,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) {
// Check for non-packed repeated fallback:
if ( decoded_wiretype ! = WireFormatLite : : WIRETYPE_LENGTH_DELIMITED ) {
PROTOBUF_MUSTTAIL return MpRepeatedVarint ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpRepeatedVarint < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
const uint16_t xform_val = ( type_card & field_layout : : kTvMask ) ;
const bool is_zigzag = xform_val = = field_layout : : kTvZigZag ;
@ -2070,14 +2085,17 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) {
// pending hasbits now:
SyncHasbits ( msg , hasbits , table ) ;
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
uint16_t rep = type_card & field_layout : : kRepMask ;
if ( rep = = field_layout : : kRep64Bits ) {
auto * field = & RefAt < RepeatedField < uint64_t > > ( msg , entry . offset ) ;
auto * field = & MaybeCreateRepeatedFieldRefAt < uint64_t , is_split > (
base , entry . offset , msg ) ;
return ctx - > ReadPackedVarint ( ptr , [ field , is_zigzag ] ( uint64_t value ) {
field - > Add ( is_zigzag ? WireFormatLite : : ZigZagDecode64 ( value ) : value ) ;
} ) ;
} else if ( rep = = field_layout : : kRep32Bits ) {
auto * field = & RefAt < RepeatedField < uint32_t > > ( msg , entry . offset ) ;
auto * field = & MaybeCreateRepeatedFieldRefAt < uint32_t , is_split > (
base , entry . offset , msg ) ;
if ( is_validated_enum ) {
const TcParseTableBase : : FieldAux aux = * table - > field_aux ( entry . aux_idx ) ;
return ctx - > ReadPackedVarint ( ptr , [ = ] ( int32_t value ) {
@ -2096,7 +2114,8 @@ PROTOBUF_NOINLINE const char* TcParser::MpPackedVarint(PROTOBUF_TC_PARAM_DECL) {
}
} else {
ABSL_DCHECK_EQ ( rep , static_cast < uint16_t > ( field_layout : : kRep8Bits ) ) ;
auto * field = & RefAt < RepeatedField < bool > > ( msg , entry . offset ) ;
auto * field =
& MaybeCreateRepeatedFieldRefAt < bool , is_split > ( base , entry . offset , msg ) ;
return ctx - > ReadPackedVarint (
ptr , [ field ] ( uint64_t value ) { field - > Add ( static_cast < bool > ( value ) ) ; } ) ;
}
@ -2146,7 +2165,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return table - > fallback ( PROTOBUF_TC_PARAM_PASS ) ;
}
if ( card = = field_layout : : kFcRepeated ) {
PROTOBUF_MUSTTAIL return MpRepeatedString ( PROTOBUF_TC_PARAM_PASS ) ;
PROTOBUF_MUSTTAIL return MpRepeatedString < is_split > ( PROTOBUF_TC_PARAM_PASS ) ;
}
const uint16_t xform_val = type_card & field_layout : : kTvMask ;
const uint16_t rep = type_card & field_layout : : kRepMask ;
@ -2222,6 +2241,7 @@ PROTOBUF_ALWAYS_INLINE const char* TcParser::ParseRepeatedStringOnce(
return ptr ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpRepeatedString (
PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
@ -2235,9 +2255,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString(
const uint16_t rep = type_card & field_layout : : kRepMask ;
const uint16_t xform_val = type_card & field_layout : : kTvMask ;
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
switch ( rep ) {
case field_layout : : kRepSString : {
auto & field = RefAt < RepeatedPtrField < std : : string > > ( msg , entry . offset ) ;
auto & field = MaybeCreateRepeatedPtrFieldRefAt < std : : string , is_split > (
base , entry . offset , msg ) ;
const char * ptr2 = ptr ;
uint32_t next_tag ;
@ -2300,10 +2322,10 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
const uint16_t rep = type_card & field_layout : : kRepMask ;
switch ( rep ) {
case field_layout : : kRepMessage :
PROTOBUF_MUSTTAIL return MpRepeatedMessageOrGroup < false > (
PROTOBUF_MUSTTAIL return MpRepeatedMessageOrGroup < is_split , false > (
PROTOBUF_TC_PARAM_PASS ) ;
case field_layout : : kRepGroup :
PROTOBUF_MUSTTAIL return MpRepeatedMessageOrGroup < true > (
PROTOBUF_MUSTTAIL return MpRepeatedMessageOrGroup < is_split , true > (
PROTOBUF_TC_PARAM_PASS ) ;
default :
PROTOBUF_MUSTTAIL return table - > fallback ( PROTOBUF_TC_PARAM_PASS ) ;
@ -2372,7 +2394,7 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
}
}
template < bool is_group >
template < bool is_split , bool is_ group >
const char * TcParser : : MpRepeatedMessageOrGroup ( PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
const uint16_t type_card = entry . type_card ;
@ -2396,7 +2418,10 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) {
}
}
auto & field = RefAt < RepeatedPtrFieldBase > ( msg , entry . offset ) ;
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
RepeatedPtrFieldBase & field =
MaybeCreateRepeatedRefAt < RepeatedPtrFieldBase , is_split > (
base , entry . offset , msg ) ;
const auto aux = * table - > field_aux ( & entry ) ;
if ( ( type_card & field_layout : : kTvMask ) = = field_layout : : kTvTable ) {
auto * inner_table = aux . table ;
@ -2405,7 +2430,7 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) {
uint32_t next_tag ;
do {
MessageLite * value =
field . Add < GenericTypeHandler < MessageLite > > ( default_instance ) ;
field . template Add < GenericTypeHandler < MessageLite > > ( default_instance ) ;
ptr = is_group ? ctx - > ParseGroup < TcParser > ( value , ptr2 , decoded_tag ,
inner_table )
: ctx - > ParseMessage < TcParser > ( value , ptr2 , inner_table ) ;
@ -2427,7 +2452,7 @@ const char* TcParser::MpRepeatedMessageOrGroup(PROTOBUF_TC_PARAM_DECL) {
uint32_t next_tag ;
do {
MessageLite * value =
field . Add < GenericTypeHandler < MessageLite > > ( default_instance ) ;
field . template Add < GenericTypeHandler < MessageLite > > ( default_instance ) ;
ptr = is_group ? ctx - > ParseGroup ( value , ptr2 , decoded_tag )
: ctx - > ParseMessage ( value , ptr2 ) ;
if ( PROTOBUF_PREDICT_FALSE ( ptr = = nullptr ) ) goto error ;
@ -2683,6 +2708,7 @@ const char* TcParser::ParseOneMapEntry(
return ptr ;
}
template < bool is_split >
PROTOBUF_NOINLINE const char * TcParser : : MpMap ( PROTOBUF_TC_PARAM_DECL ) {
const auto & entry = RefAt < FieldEntry > ( table , data . entry_offset ( ) ) ;
// `aux[0]` points into a MapAuxInfo.
@ -2702,10 +2728,11 @@ PROTOBUF_NOINLINE const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) {
// Otherwise, it points into a MapField and we must synchronize with
// reflection. It is done by calling the MutableMap() virtual function on the
// field's base class.
void * const base = MaybeGetSplitBase ( msg , is_split , table ) ;
UntypedMapBase & map =
map_info . use_lite
? RefAt < UntypedMapBase > ( msg , entry . offset )
: * RefAt < MapFieldBaseForParse > ( msg , entry . offset ) . MutableMap ( ) ;
? RefAt < UntypedMapBase > ( base , entry . offset )
: * RefAt < MapFieldBaseForParse > ( base , entry . offset ) . MutableMap ( ) ;
const uint32_t saved_tag = data . tag ( ) ;