Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity
Protocol Buffers - Google's data interchange format (grpc依赖) https://developers.google.com/protocol-buffers/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
16011 Commits
201 Branches
346 Tags
254 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 0883b22ac4
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '0883b22ac4'
${ noResults }
protobuf/upb/wire/decode_fast.c

996 lines
49 KiB
Raw Blame History

// Protocol Buffers - Google's data interchange format
// Copyright 2023 Google LLC. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
// Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64.
// Also the table size grows by 2x.
//
// Could potentially be ported to other 64-bit archs that pass at least six
// arguments in registers and have 8 unused high bits in pointers.
//
// The overall design is to create specialized functions for every possible
// field type (eg. oneof boolean field with a 1 byte tag) and then dispatch
// to the specialized function as quickly as possible.
#include "upb/wire/decode_fast.h"
#include "upb/message/array.h"
#include "upb/message/internal/array.h"
#include "upb/message/internal/types.h"
#include "upb/wire/internal/decode.h"
// Must be last.
#include "upb/port/def.inc"
#if UPB_FASTTABLE
// The standard set of arguments passed to each parsing function.
// Thanks to x86-64 calling conventions, these will stay in registers.
#define UPB_PARSE_PARAMS \
upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \
uint64_t hasbits, uint64_t data
#define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data
#define RETURN_GENERIC(m) \
/* Uncomment either of these for debugging purposes. */ \
/* fprintf(stderr, m); */ \
/*__builtin_trap(); */ \
return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0);
typedef enum {
CARD_s = 0, /* Singular (optional, non-repeated) */
CARD_o = 1, /* Oneof */
CARD_r = 2, /* Repeated */
CARD_p = 3 /* Packed Repeated */
} upb_card;
UPB_NOINLINE
static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) {
int overrun = data;
ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline(
&d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback);
data = _upb_FastDecoder_LoadTag(ptr);
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
}
UPB_FORCEINLINE
static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) {
int overrun;
switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) {
case kUpb_IsDoneStatus_Done:
*(uint32_t*)msg |= hasbits; // Sync hasbits.
const upb_MiniTable* l = decode_totablep(table);
return UPB_UNLIKELY(l->required_count)
? _upb_Decoder_CheckRequired(d, ptr, msg, l)
: ptr;
case kUpb_IsDoneStatus_NotDone:
break;
case kUpb_IsDoneStatus_NeedFallback:
data = overrun;
UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS);
}
// Read two bytes of tag data (for a one-byte tag, the high byte is junk).
data = _upb_FastDecoder_LoadTag(ptr);
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
}
UPB_FORCEINLINE
static bool fastdecode_checktag(uint16_t data, int tagbytes) {
if (tagbytes == 1) {
return (data & 0xff) == 0;
} else {
return data == 0;
}
}
UPB_FORCEINLINE
static const char* fastdecode_longsize(const char* ptr, int* size) {
int i;
UPB_ASSERT(*size & 0x80);
*size &= 0xff;
for (i = 0; i < 3; i++) {
ptr++;
size_t byte = (uint8_t)ptr[-1];
*size += (byte - 1) << (7 + 7 * i);
if (UPB_LIKELY((byte & 0x80) == 0)) return ptr;
}
ptr++;
size_t byte = (uint8_t)ptr[-1];
// len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected
// for a 32 bit varint.
if (UPB_UNLIKELY(byte >= 8)) return NULL;
*size += (byte - 1) << 28;
return ptr;
}
UPB_FORCEINLINE
static const char* fastdecode_delimited(
upb_Decoder* d, const char* ptr,
upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) {
ptr++;
// Sign-extend so varint greater than one byte becomes negative, causing
// fast delimited parse to fail.
int len = (int8_t)ptr[-1];
if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func,
ctx)) {
// Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer.
// If it exceeds the buffer limit, limit/limit_ptr will change during
// sub-message parsing, so we need to preserve delta, not limit.
if (UPB_UNLIKELY(len & 0x80)) {
// Size varint >1 byte (length >= 128).
ptr = fastdecode_longsize(ptr, &len);
if (!ptr) {
// Corrupt wire format: size exceeded INT_MAX.
return NULL;
}
}
if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) {
// Corrupt wire format: invalid limit.
return NULL;
}
int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len);
ptr = func(&d->input, ptr, ctx);
upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta);
}
return ptr;
}
/* singular, oneof, repeated field handling ***********************************/
typedef struct {
upb_Array* arr;
void* end;
} fastdecode_arr;
typedef enum {
FD_NEXT_ATLIMIT,
FD_NEXT_SAMEFIELD,
FD_NEXT_OTHERFIELD
} fastdecode_next;
typedef struct {
void* dst;
fastdecode_next next;
uint32_t tag;
} fastdecode_nextret;
UPB_FORCEINLINE
static void* fastdecode_resizearr(upb_Decoder* d, void* dst,
fastdecode_arr* farr, int valbytes) {
if (UPB_UNLIKELY(dst == farr->end)) {
size_t old_size = farr->arr->capacity;
size_t old_bytes = old_size * valbytes;
size_t new_size = old_size * 2;
size_t new_bytes = new_size * valbytes;
char* old_ptr = _upb_array_ptr(farr->arr);
char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes);
uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
farr->arr->capacity = new_size;
farr->arr->data = _upb_array_tagptr(new_ptr, elem_size_lg2);
dst = (void*)(new_ptr + (old_size * valbytes));
farr->end = (void*)(new_ptr + (new_size * valbytes));
}
return dst;
}
UPB_FORCEINLINE
static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) {
if (tagbytes == 1) {
return (uint8_t)tag == (uint8_t)data;
} else {
return (uint16_t)tag == (uint16_t)data;
}
}
UPB_FORCEINLINE
static void fastdecode_commitarr(void* dst, fastdecode_arr* farr,
int valbytes) {
farr->arr->size =
(size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes;
}
UPB_FORCEINLINE
static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst,
const char** ptr,
fastdecode_arr* farr,
uint64_t data, int tagbytes,
int valbytes) {
fastdecode_nextret ret;
dst = (char*)dst + valbytes;
if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) {
ret.tag = _upb_FastDecoder_LoadTag(*ptr);
if (fastdecode_tagmatch(ret.tag, data, tagbytes)) {
ret.next = FD_NEXT_SAMEFIELD;
} else {
fastdecode_commitarr(dst, farr, valbytes);
ret.next = FD_NEXT_OTHERFIELD;
}
} else {
fastdecode_commitarr(dst, farr, valbytes);
ret.next = FD_NEXT_ATLIMIT;
}
ret.dst = dst;
return ret;
}
UPB_FORCEINLINE
static void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) {
size_t ofs = data >> 48;
return (char*)msg + ofs;
}
UPB_FORCEINLINE
static void* fastdecode_getfield(upb_Decoder* d, const char* ptr,
upb_Message* msg, uint64_t* data,
uint64_t* hasbits, fastdecode_arr* farr,
int valbytes, upb_card card) {
switch (card) {
case CARD_s: {
uint8_t hasbit_index = *data >> 24;
// Set hasbit and return pointer to scalar field.
*hasbits |= 1ull << hasbit_index;
return fastdecode_fieldmem(msg, *data);
}
case CARD_o: {
uint16_t case_ofs = *data >> 32;
uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t);
uint8_t field_number = *data >> 24;
*oneof_case = field_number;
return fastdecode_fieldmem(msg, *data);
}
case CARD_r: {
// Get pointer to upb_Array and allocate/expand if necessary.
uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
upb_Array** arr_p = fastdecode_fieldmem(msg, *data);
char* begin;
*(uint32_t*)msg |= *hasbits;
*hasbits = 0;
if (UPB_LIKELY(!*arr_p)) {
farr->arr = _upb_Array_New(&d->arena, 8, elem_size_lg2);
*arr_p = farr->arr;
} else {
farr->arr = *arr_p;
}
begin = _upb_array_ptr(farr->arr);
farr->end = begin + (farr->arr->capacity * valbytes);
*data = _upb_FastDecoder_LoadTag(ptr);
return begin + (farr->arr->size * valbytes);
}
default:
UPB_UNREACHABLE();
}
}
UPB_FORCEINLINE
static bool fastdecode_flippacked(uint64_t* data, int tagbytes) {
*data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype.
return fastdecode_checktag(*data, tagbytes);
}
#define FASTDECODE_CHECKPACKED(tagbytes, card, func) \
if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \
UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \
} \
RETURN_GENERIC("packed check tag mismatch\n"); \
}
/* varint fields **************************************************************/
UPB_FORCEINLINE
static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) {
if (valbytes == 1) {
return val != 0;
} else if (zigzag) {
if (valbytes == 4) {
uint32_t n = val;
return (n >> 1) ^ -(int32_t)(n & 1);
} else if (valbytes == 8) {
return (val >> 1) ^ -(int64_t)(val & 1);
}
UPB_UNREACHABLE();
}
return val;
}
UPB_FORCEINLINE
static const char* fastdecode_varint64(const char* ptr, uint64_t* val) {
ptr++;
*val = (uint8_t)ptr[-1];
if (UPB_UNLIKELY(*val & 0x80)) {
int i;
for (i = 0; i < 8; i++) {
ptr++;
uint64_t byte = (uint8_t)ptr[-1];
*val += (byte - 1) << (7 + 7 * i);
if (UPB_LIKELY((byte & 0x80) == 0)) goto done;
}
ptr++;
uint64_t byte = (uint8_t)ptr[-1];
if (byte > 1) {
return NULL;
}
*val += (byte - 1) << 63;
}
done:
UPB_ASSUME(ptr != NULL);
return ptr;
}
#define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, zigzag, packed) \
uint64_t val; \
void* dst; \
fastdecode_arr farr; \
\
FASTDECODE_CHECKPACKED(tagbytes, card, packed); \
\
dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
card); \
if (card == CARD_r) { \
if (UPB_UNLIKELY(!dst)) { \
RETURN_GENERIC("need array resize\n"); \
} \
} \
\
again: \
if (card == CARD_r) { \
dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
} \
\
ptr += tagbytes; \
ptr = fastdecode_varint64(ptr, &val); \
if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
val = fastdecode_munge(val, valbytes, zigzag); \
memcpy(dst, &val, valbytes); \
\
if (card == CARD_r) { \
fastdecode_nextret ret = fastdecode_nextrepeated( \
d, dst, &ptr, &farr, data, tagbytes, valbytes); \
switch (ret.next) { \
case FD_NEXT_SAMEFIELD: \
dst = ret.dst; \
goto again; \
case FD_NEXT_OTHERFIELD: \
data = ret.tag; \
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
case FD_NEXT_ATLIMIT: \
return ptr; \
} \
} \
\
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
typedef struct {
uint8_t valbytes;
bool zigzag;
void* dst;
fastdecode_arr farr;
} fastdecode_varintdata;
UPB_FORCEINLINE
static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e,
const char* ptr, void* ctx) {
upb_Decoder* d = (upb_Decoder*)e;
fastdecode_varintdata* data = ctx;
void* dst = data->dst;
uint64_t val;
while (!_upb_Decoder_IsDone(d, &ptr)) {
dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes);
ptr = fastdecode_varint64(ptr, &val);
if (ptr == NULL) return NULL;
val = fastdecode_munge(val, data->valbytes, data->zigzag);
memcpy(dst, &val, data->valbytes);
dst = (char*)dst + data->valbytes;
}
fastdecode_commitarr(dst, &data->farr, data->valbytes);
return ptr;
}
#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, zigzag, unpacked) \
fastdecode_varintdata ctx = {valbytes, zigzag}; \
\
FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \
\
ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \
valbytes, CARD_r); \
if (UPB_UNLIKELY(!ctx.dst)) { \
RETURN_GENERIC("need array resize\n"); \
} \
\
ptr += tagbytes; \
ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \
\
if (UPB_UNLIKELY(ptr == NULL)) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
} \
\
UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0);
#define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, zigzag, unpacked, packed) \
if (card == CARD_p) { \
FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, zigzag, unpacked); \
} else { \
FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, zigzag, packed); \
}
#define z_ZZ true
#define b_ZZ false
#define v_ZZ false
/* Generate all combinations:
* {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */
#define F(card, type, valbytes, tagbytes) \
UPB_NOINLINE \
const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
CARD_##card, type##_ZZ, \
upb_pr##type##valbytes##_##tagbytes##bt, \
upb_pp##type##valbytes##_##tagbytes##bt); \
}
#define TYPES(card, tagbytes) \
F(card, b, 1, tagbytes) \
F(card, v, 4, tagbytes) \
F(card, v, 8, tagbytes) \
F(card, z, 4, tagbytes) \
F(card, z, 8, tagbytes)
#define TAGBYTES(card) \
TYPES(card, 1) \
TYPES(card, 2)
TAGBYTES(s)
TAGBYTES(o)
TAGBYTES(r)
TAGBYTES(p)
#undef z_ZZ
#undef b_ZZ
#undef v_ZZ
#undef o_ONEOF
#undef s_ONEOF
#undef r_ONEOF
#undef F
#undef TYPES
#undef TAGBYTES
#undef FASTDECODE_UNPACKEDVARINT
#undef FASTDECODE_PACKEDVARINT
#undef FASTDECODE_VARINT
/* fixed fields ***************************************************************/
#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, packed) \
void* dst; \
fastdecode_arr farr; \
\
FASTDECODE_CHECKPACKED(tagbytes, card, packed) \
\
dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
card); \
if (card == CARD_r) { \
if (UPB_UNLIKELY(!dst)) { \
RETURN_GENERIC("couldn't allocate array in arena\n"); \
} \
} \
\
again: \
if (card == CARD_r) { \
dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
} \
\
ptr += tagbytes; \
memcpy(dst, ptr, valbytes); \
ptr += valbytes; \
\
if (card == CARD_r) { \
fastdecode_nextret ret = fastdecode_nextrepeated( \
d, dst, &ptr, &farr, data, tagbytes, valbytes); \
switch (ret.next) { \
case FD_NEXT_SAMEFIELD: \
dst = ret.dst; \
goto again; \
case FD_NEXT_OTHERFIELD: \
data = ret.tag; \
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
case FD_NEXT_ATLIMIT: \
return ptr; \
} \
} \
\
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, unpacked) \
FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \
\
ptr += tagbytes; \
int size = (uint8_t)ptr[0]; \
ptr++; \
if (size & 0x80) { \
ptr = fastdecode_longsize(ptr, &size); \
} \
\
if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \
&d->input, ptr, size) || \
(size % valbytes) != 0)) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
} \
\
upb_Array** arr_p = fastdecode_fieldmem(msg, data); \
upb_Array* arr = *arr_p; \
uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \
int elems = size / valbytes; \
\
if (UPB_LIKELY(!arr)) { \
*arr_p = arr = _upb_Array_New(&d->arena, elems, elem_size_lg2); \
if (!arr) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
} \
} else { \
_upb_Array_ResizeUninitialized(arr, elems, &d->arena); \
} \
\
char* dst = _upb_array_ptr(arr); \
memcpy(dst, ptr, size); \
arr->size = elems; \
\
ptr += size; \
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
#define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, unpacked, packed) \
if (card == CARD_p) { \
FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, unpacked); \
} else { \
FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
valbytes, card, packed); \
}
/* Generate all combinations:
* {s,o,r,p} x {f4,f8} x {1bt,2bt} */
#define F(card, valbytes, tagbytes) \
UPB_NOINLINE \
const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \
upb_prf##valbytes##_##tagbytes##bt); \
}
#define TYPES(card, tagbytes) \
F(card, 4, tagbytes) \
F(card, 8, tagbytes)
#define TAGBYTES(card) \
TYPES(card, 1) \
TYPES(card, 2)
TAGBYTES(s)
TAGBYTES(o)
TAGBYTES(r)
TAGBYTES(p)
#undef F
#undef TYPES
#undef TAGBYTES
#undef FASTDECODE_UNPACKEDFIXED
#undef FASTDECODE_PACKEDFIXED
/* string fields **************************************************************/
typedef const char* fastdecode_copystr_func(struct upb_Decoder* d,
const char* ptr, upb_Message* msg,
const upb_MiniTable* table,
uint64_t hasbits,
upb_StringView* dst);
UPB_NOINLINE
static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr,
upb_Message* msg, intptr_t table,
uint64_t hasbits, uint64_t data) {
upb_StringView* dst = (upb_StringView*)data;
if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);
}
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
}
#define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \
int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \
ptr++; \
if (size & 0x80) { \
ptr = fastdecode_longsize(ptr, &size); \
} \
\
if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \
dst->size = 0; \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
} \
\
const char* s_ptr = ptr; \
ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \
if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \
dst->data = s_ptr; \
dst->size = size; \
\
if (validate_utf8) { \
data = (uint64_t)dst; \
UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
} else { \
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
}
UPB_NOINLINE
static const char* fastdecode_longstring_utf8(struct upb_Decoder* d,
const char* ptr, upb_Message* msg,
intptr_t table, uint64_t hasbits,
uint64_t data) {
upb_StringView* dst = (upb_StringView*)data;
FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true);
}
UPB_NOINLINE
static const char* fastdecode_longstring_noutf8(
struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table,
uint64_t hasbits, uint64_t data) {
upb_StringView* dst = (upb_StringView*)data;
FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false);
}
UPB_FORCEINLINE
static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
int copy, char* data, upb_StringView* dst) {
d->arena.head.ptr += copy;
dst->data = data;
UPB_UNPOISON_MEMORY_REGION(data, copy);
memcpy(data, ptr, copy);
UPB_POISON_MEMORY_REGION(data + size, copy - size);
}
#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
card, validate_utf8) \
upb_StringView* dst; \
fastdecode_arr farr; \
int64_t size; \
size_t arena_has; \
size_t common_has; \
char* buf; \
\
UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \
UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \
\
dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
sizeof(upb_StringView), card); \
\
again: \
if (card == CARD_r) { \
dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
} \
\
size = (uint8_t)ptr[tagbytes]; \
ptr += tagbytes + 1; \
dst->size = size; \
\
buf = d->arena.head.ptr; \
arena_has = _upb_ArenaHas(&d->arena); \
common_has = UPB_MIN(arena_has, \
upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
\
if (UPB_LIKELY(size <= 15 - tagbytes)) { \
if (arena_has < 16) goto longstr; \
d->arena.head.ptr += 16; \
memcpy(buf, ptr - tagbytes - 1, 16); \
dst->data = buf + tagbytes + 1; \
} else if (UPB_LIKELY(size <= 32)) { \
if (UPB_UNLIKELY(common_has < 32)) goto longstr; \
fastdecode_docopy(d, ptr, size, 32, buf, dst); \
} else if (UPB_LIKELY(size <= 64)) { \
if (UPB_UNLIKELY(common_has < 64)) goto longstr; \
fastdecode_docopy(d, ptr, size, 64, buf, dst); \
} else if (UPB_LIKELY(size < 128)) { \
if (UPB_UNLIKELY(common_has < 128)) goto longstr; \
fastdecode_docopy(d, ptr, size, 128, buf, dst); \
} else { \
goto longstr; \
} \
\
ptr += size; \
\
if (card == CARD_r) { \
if (validate_utf8 && \
!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
} \
fastdecode_nextret ret = fastdecode_nextrepeated( \
d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
switch (ret.next) { \
case FD_NEXT_SAMEFIELD: \
dst = ret.dst; \
goto again; \
case FD_NEXT_OTHERFIELD: \
data = ret.tag; \
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
case FD_NEXT_ATLIMIT: \
return ptr; \
} \
} \
\
if (card != CARD_r && validate_utf8) { \
data = (uint64_t)dst; \
UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
} \
\
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
\
longstr: \
if (card == CARD_r) { \
fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \
} \
ptr--; \
if (validate_utf8) { \
UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \
hasbits, (uint64_t)dst); \
} else { \
UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \
hasbits, (uint64_t)dst); \
}
#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \
copyfunc, validate_utf8) \
upb_StringView* dst; \
fastdecode_arr farr; \
int64_t size; \
\
if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
RETURN_GENERIC("string field tag mismatch\n"); \
} \
\
if (UPB_UNLIKELY( \
!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \
UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \
} \
\
dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
sizeof(upb_StringView), card); \
\
again: \
if (card == CARD_r) { \
dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
} \
\
size = (int8_t)ptr[tagbytes]; \
ptr += tagbytes + 1; \
\
if (UPB_UNLIKELY( \
!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \
ptr--; \
if (validate_utf8) { \
return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \
(uint64_t)dst); \
} else { \
return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \
(uint64_t)dst); \
} \
} \
\
dst->data = ptr; \
dst->size = size; \
ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \
dst->size); \
\
if (card == CARD_r) { \
if (validate_utf8 && \
!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
} \
fastdecode_nextret ret = fastdecode_nextrepeated( \
d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
switch (ret.next) { \
case FD_NEXT_SAMEFIELD: \
dst = ret.dst; \
goto again; \
case FD_NEXT_OTHERFIELD: \
data = ret.tag; \
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
case FD_NEXT_ATLIMIT: \
return ptr; \
} \
} \
\
if (card != CARD_r && validate_utf8) { \
data = (uint64_t)dst; \
UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
} \
\
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
/* Generate all combinations:
* {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */
#define s_VALIDATE true
#define b_VALIDATE false
#define F(card, tagbytes, type) \
UPB_NOINLINE \
const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
CARD_##card, type##_VALIDATE); \
} \
const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \
CARD_##card, upb_c##card##type##_##tagbytes##bt, \
type##_VALIDATE); \
}
#define UTF8(card, tagbytes) \
F(card, tagbytes, s) \
F(card, tagbytes, b)
#define TAGBYTES(card) \
UTF8(card, 1) \
UTF8(card, 2)
TAGBYTES(s)
TAGBYTES(o)
TAGBYTES(r)
#undef s_VALIDATE
#undef b_VALIDATE
#undef F
#undef TAGBYTES
#undef FASTDECODE_LONGSTRING
#undef FASTDECODE_COPYSTRING
#undef FASTDECODE_STRING
/* message fields *************************************************************/
UPB_INLINE
upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* l,
int msg_ceil_bytes) {
size_t size = l->size + sizeof(upb_Message_Internal);
char* msg_data;
if (UPB_LIKELY(msg_ceil_bytes > 0 &&
_upb_ArenaHas(&d->arena) >= msg_ceil_bytes)) {
UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
msg_data = d->arena.head.ptr;
d->arena.head.ptr += size;
UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
memset(msg_data, 0, msg_ceil_bytes);
UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
} else {
msg_data = (char*)upb_Arena_Malloc(&d->arena, size);
memset(msg_data, 0, size);
}
return msg_data + sizeof(upb_Message_Internal);
}
typedef struct {
intptr_t table;
upb_Message* msg;
} fastdecode_submsgdata;
UPB_FORCEINLINE
static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e,
const char* ptr, void* ctx) {
upb_Decoder* d = (upb_Decoder*)e;
fastdecode_submsgdata* submsg = ctx;
ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0);
UPB_ASSUME(ptr != NULL);
return ptr;
}
#define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \
msg_ceil_bytes, card) \
\
if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
RETURN_GENERIC("submessage field tag mismatch\n"); \
} \
\
if (--d->depth == 0) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \
} \
\
upb_Message** dst; \
uint32_t submsg_idx = (data >> 16) & 0xff; \
const upb_MiniTable* tablep = decode_totablep(table); \
const upb_MiniTable* subtablep = tablep->subs[submsg_idx].submsg; \
fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \
fastdecode_arr farr; \
\
if (subtablep->table_mask == (uint8_t)-1) { \
RETURN_GENERIC("submessage doesn't have fast tables."); \
} \
\
dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
sizeof(upb_Message*), card); \
\
if (card == CARD_s) { \
*(uint32_t*)msg |= hasbits; \
hasbits = 0; \
} \
\
again: \
if (card == CARD_r) { \
dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \
} \
\
submsg.msg = *dst; \
\
if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \
*dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \
} \
\
ptr += tagbytes; \
ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \
\
if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \
_upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
} \
\
if (card == CARD_r) { \
fastdecode_nextret ret = fastdecode_nextrepeated( \
d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \
switch (ret.next) { \
case FD_NEXT_SAMEFIELD: \
dst = ret.dst; \
goto again; \
case FD_NEXT_OTHERFIELD: \
d->depth++; \
data = ret.tag; \
UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
case FD_NEXT_ATLIMIT: \
d->depth++; \
return ptr; \
} \
} \
\
d->depth++; \
UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
#define F(card, tagbytes, size_ceil, ceil_arg) \
const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \
UPB_PARSE_PARAMS) { \
FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \
CARD_##card); \
}
#define SIZES(card, tagbytes) \
F(card, tagbytes, 64, 64) \
F(card, tagbytes, 128, 128) \
F(card, tagbytes, 192, 192) \
F(card, tagbytes, 256, 256) \
F(card, tagbytes, max, -1)
#define TAGBYTES(card) \
SIZES(card, 1) \
SIZES(card, 2)
TAGBYTES(s)
TAGBYTES(o)
TAGBYTES(r)
#undef TAGBYTES
#undef SIZES
#undef F
#undef FASTDECODE_SUBMSG
#endif /* UPB_FASTTABLE */