// Protocol Buffers - Google's data interchange format // Copyright 2023 Google LLC. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd // Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64. // Also the table size grows by 2x. // // Could potentially be ported to other 64-bit archs that pass at least six // arguments in registers and have 8 unused high bits in pointers. // // The overall design is to create specialized functions for every possible // field type (eg. oneof boolean field with a 1 byte tag) and then dispatch // to the specialized function as quickly as possible. #include "upb/wire/decode_fast.h" #include "upb/message/array.h" #include "upb/message/internal/array.h" #include "upb/message/internal/types.h" #include "upb/mini_table/sub.h" #include "upb/wire/internal/decode.h" // Must be last. #include "upb/port/def.inc" #if UPB_FASTTABLE // The standard set of arguments passed to each parsing function. // Thanks to x86-64 calling conventions, these will stay in registers. #define UPB_PARSE_PARAMS \ upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \ uint64_t hasbits, uint64_t data #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data #define RETURN_GENERIC(m) \ /* Uncomment either of these for debugging purposes. */ \ /* fprintf(stderr, m); */ \ /*__builtin_trap(); */ \ return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0); typedef enum { CARD_s = 0, /* Singular (optional, non-repeated) */ CARD_o = 1, /* Oneof */ CARD_r = 2, /* Repeated */ CARD_p = 3 /* Packed Repeated */ } upb_card; UPB_NOINLINE static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) { int overrun = data; ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline( &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback); data = _upb_FastDecoder_LoadTag(ptr); UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); } UPB_FORCEINLINE static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) { int overrun; switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) { case kUpb_IsDoneStatus_Done: *(uint32_t*)msg |= hasbits; // Sync hasbits. const upb_MiniTable* m = decode_totablep(table); return UPB_UNLIKELY(m->UPB_PRIVATE(required_count)) ? _upb_Decoder_CheckRequired(d, ptr, msg, m) : ptr; case kUpb_IsDoneStatus_NotDone: break; case kUpb_IsDoneStatus_NeedFallback: data = overrun; UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS); } // Read two bytes of tag data (for a one-byte tag, the high byte is junk). data = _upb_FastDecoder_LoadTag(ptr); UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); } UPB_FORCEINLINE static bool fastdecode_checktag(uint16_t data, int tagbytes) { if (tagbytes == 1) { return (data & 0xff) == 0; } else { return data == 0; } } UPB_FORCEINLINE static const char* fastdecode_longsize(const char* ptr, int* size) { int i; UPB_ASSERT(*size & 0x80); *size &= 0xff; for (i = 0; i < 3; i++) { ptr++; size_t byte = (uint8_t)ptr[-1]; *size += (byte - 1) << (7 + 7 * i); if (UPB_LIKELY((byte & 0x80) == 0)) return ptr; } ptr++; size_t byte = (uint8_t)ptr[-1]; // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected // for a 32 bit varint. if (UPB_UNLIKELY(byte >= 8)) return NULL; *size += (byte - 1) << 28; return ptr; } UPB_FORCEINLINE static const char* fastdecode_delimited( upb_Decoder* d, const char* ptr, upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) { ptr++; // Sign-extend so varint greater than one byte becomes negative, causing // fast delimited parse to fail. int len = (int8_t)ptr[-1]; if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func, ctx)) { // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer. // If it exceeds the buffer limit, limit/limit_ptr will change during // sub-message parsing, so we need to preserve delta, not limit. if (UPB_UNLIKELY(len & 0x80)) { // Size varint >1 byte (length >= 128). ptr = fastdecode_longsize(ptr, &len); if (!ptr) { // Corrupt wire format: size exceeded INT_MAX. return NULL; } } if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) { // Corrupt wire format: invalid limit. return NULL; } int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len); ptr = func(&d->input, ptr, ctx); upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta); } return ptr; } /* singular, oneof, repeated field handling ***********************************/ typedef struct { upb_Array* arr; void* end; } fastdecode_arr; typedef enum { FD_NEXT_ATLIMIT, FD_NEXT_SAMEFIELD, FD_NEXT_OTHERFIELD } fastdecode_next; typedef struct { void* dst; fastdecode_next next; uint32_t tag; } fastdecode_nextret; UPB_FORCEINLINE static void* fastdecode_resizearr(upb_Decoder* d, void* dst, fastdecode_arr* farr, int valbytes) { if (UPB_UNLIKELY(dst == farr->end)) { size_t old_capacity = farr->arr->UPB_PRIVATE(capacity); size_t old_bytes = old_capacity * valbytes; size_t new_capacity = old_capacity * 2; size_t new_bytes = new_capacity * valbytes; char* old_ptr = _upb_array_ptr(farr->arr); char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes); uint8_t elem_size_lg2 = __builtin_ctz(valbytes); UPB_PRIVATE(_upb_Array_SetTaggedPtr)(farr->arr, new_ptr, elem_size_lg2); farr->arr->UPB_PRIVATE(capacity) = new_capacity; dst = (void*)(new_ptr + (old_capacity * valbytes)); farr->end = (void*)(new_ptr + (new_capacity * valbytes)); } return dst; } UPB_FORCEINLINE static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) { if (tagbytes == 1) { return (uint8_t)tag == (uint8_t)data; } else { return (uint16_t)tag == (uint16_t)data; } } UPB_FORCEINLINE static void fastdecode_commitarr(void* dst, fastdecode_arr* farr, int valbytes) { farr->arr->UPB_PRIVATE(size) = (size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes; } UPB_FORCEINLINE static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst, const char** ptr, fastdecode_arr* farr, uint64_t data, int tagbytes, int valbytes) { fastdecode_nextret ret; dst = (char*)dst + valbytes; if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) { ret.tag = _upb_FastDecoder_LoadTag(*ptr); if (fastdecode_tagmatch(ret.tag, data, tagbytes)) { ret.next = FD_NEXT_SAMEFIELD; } else { fastdecode_commitarr(dst, farr, valbytes); ret.next = FD_NEXT_OTHERFIELD; } } else { fastdecode_commitarr(dst, farr, valbytes); ret.next = FD_NEXT_ATLIMIT; } ret.dst = dst; return ret; } UPB_FORCEINLINE static void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) { size_t ofs = data >> 48; return (char*)msg + ofs; } UPB_FORCEINLINE static void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg, uint64_t* data, uint64_t* hasbits, fastdecode_arr* farr, int valbytes, upb_card card) { switch (card) { case CARD_s: { uint8_t hasbit_index = *data >> 24; // Set hasbit and return pointer to scalar field. *hasbits |= 1ull << hasbit_index; return fastdecode_fieldmem(msg, *data); } case CARD_o: { uint16_t case_ofs = *data >> 32; uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t); uint8_t field_number = *data >> 24; *oneof_case = field_number; return fastdecode_fieldmem(msg, *data); } case CARD_r: { // Get pointer to upb_Array and allocate/expand if necessary. uint8_t elem_size_lg2 = __builtin_ctz(valbytes); upb_Array** arr_p = fastdecode_fieldmem(msg, *data); char* begin; *(uint32_t*)msg |= *hasbits; *hasbits = 0; if (UPB_LIKELY(!*arr_p)) { farr->arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, 8, elem_size_lg2); *arr_p = farr->arr; } else { farr->arr = *arr_p; } begin = _upb_array_ptr(farr->arr); farr->end = begin + (farr->arr->UPB_PRIVATE(capacity) * valbytes); *data = _upb_FastDecoder_LoadTag(ptr); return begin + (farr->arr->UPB_PRIVATE(size) * valbytes); } default: UPB_UNREACHABLE(); } } UPB_FORCEINLINE static bool fastdecode_flippacked(uint64_t* data, int tagbytes) { *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype. return fastdecode_checktag(*data, tagbytes); } #define FASTDECODE_CHECKPACKED(tagbytes, card, func) \ if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \ } \ RETURN_GENERIC("packed check tag mismatch\n"); \ } /* varint fields **************************************************************/ UPB_FORCEINLINE static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) { if (valbytes == 1) { return val != 0; } else if (zigzag) { if (valbytes == 4) { uint32_t n = val; return (n >> 1) ^ -(int32_t)(n & 1); } else if (valbytes == 8) { return (val >> 1) ^ -(int64_t)(val & 1); } UPB_UNREACHABLE(); } return val; } UPB_FORCEINLINE static const char* fastdecode_varint64(const char* ptr, uint64_t* val) { ptr++; *val = (uint8_t)ptr[-1]; if (UPB_UNLIKELY(*val & 0x80)) { int i; for (i = 0; i < 8; i++) { ptr++; uint64_t byte = (uint8_t)ptr[-1]; *val += (byte - 1) << (7 + 7 * i); if (UPB_LIKELY((byte & 0x80) == 0)) goto done; } ptr++; uint64_t byte = (uint8_t)ptr[-1]; if (byte > 1) { return NULL; } *val += (byte - 1) << 63; } done: UPB_ASSUME(ptr != NULL); return ptr; } #define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, zigzag, packed) \ uint64_t val; \ void* dst; \ fastdecode_arr farr; \ \ FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ card); \ if (card == CARD_r) { \ if (UPB_UNLIKELY(!dst)) { \ RETURN_GENERIC("need array resize\n"); \ } \ } \ \ again: \ if (card == CARD_r) { \ dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ } \ \ ptr += tagbytes; \ ptr = fastdecode_varint64(ptr, &val); \ if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ val = fastdecode_munge(val, valbytes, zigzag); \ memcpy(dst, &val, valbytes); \ \ if (card == CARD_r) { \ fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, valbytes); \ switch (ret.next) { \ case FD_NEXT_SAMEFIELD: \ dst = ret.dst; \ goto again; \ case FD_NEXT_OTHERFIELD: \ data = ret.tag; \ UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ case FD_NEXT_ATLIMIT: \ return ptr; \ } \ } \ \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); typedef struct { uint8_t valbytes; bool zigzag; void* dst; fastdecode_arr farr; } fastdecode_varintdata; UPB_FORCEINLINE static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e, const char* ptr, void* ctx) { upb_Decoder* d = (upb_Decoder*)e; fastdecode_varintdata* data = ctx; void* dst = data->dst; uint64_t val; while (!_upb_Decoder_IsDone(d, &ptr)) { dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes); ptr = fastdecode_varint64(ptr, &val); if (ptr == NULL) return NULL; val = fastdecode_munge(val, data->valbytes, data->zigzag); memcpy(dst, &val, data->valbytes); dst = (char*)dst + data->valbytes; } fastdecode_commitarr(dst, &data->farr, data->valbytes); return ptr; } #define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, zigzag, unpacked) \ fastdecode_varintdata ctx = {valbytes, zigzag}; \ \ FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \ \ ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ valbytes, CARD_r); \ if (UPB_UNLIKELY(!ctx.dst)) { \ RETURN_GENERIC("need array resize\n"); \ } \ \ ptr += tagbytes; \ ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ \ if (UPB_UNLIKELY(ptr == NULL)) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ } \ \ UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); #define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, zigzag, unpacked, packed) \ if (card == CARD_p) { \ FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, zigzag, unpacked); \ } else { \ FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, zigzag, packed); \ } #define z_ZZ true #define b_ZZ false #define v_ZZ false /* Generate all combinations: * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */ #define F(card, type, valbytes, tagbytes) \ UPB_NOINLINE \ const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ CARD_##card, type##_ZZ, \ upb_pr##type##valbytes##_##tagbytes##bt, \ upb_pp##type##valbytes##_##tagbytes##bt); \ } #define TYPES(card, tagbytes) \ F(card, b, 1, tagbytes) \ F(card, v, 4, tagbytes) \ F(card, v, 8, tagbytes) \ F(card, z, 4, tagbytes) \ F(card, z, 8, tagbytes) #define TAGBYTES(card) \ TYPES(card, 1) \ TYPES(card, 2) TAGBYTES(s) TAGBYTES(o) TAGBYTES(r) TAGBYTES(p) #undef z_ZZ #undef b_ZZ #undef v_ZZ #undef o_ONEOF #undef s_ONEOF #undef r_ONEOF #undef F #undef TYPES #undef TAGBYTES #undef FASTDECODE_UNPACKEDVARINT #undef FASTDECODE_PACKEDVARINT #undef FASTDECODE_VARINT /* fixed fields ***************************************************************/ #define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, packed) \ void* dst; \ fastdecode_arr farr; \ \ FASTDECODE_CHECKPACKED(tagbytes, card, packed) \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ card); \ if (card == CARD_r) { \ if (UPB_UNLIKELY(!dst)) { \ RETURN_GENERIC("couldn't allocate array in arena\n"); \ } \ } \ \ again: \ if (card == CARD_r) { \ dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ } \ \ ptr += tagbytes; \ memcpy(dst, ptr, valbytes); \ ptr += valbytes; \ \ if (card == CARD_r) { \ fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, valbytes); \ switch (ret.next) { \ case FD_NEXT_SAMEFIELD: \ dst = ret.dst; \ goto again; \ case FD_NEXT_OTHERFIELD: \ data = ret.tag; \ UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ case FD_NEXT_ATLIMIT: \ return ptr; \ } \ } \ \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, unpacked) \ FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ \ ptr += tagbytes; \ int size = (uint8_t)ptr[0]; \ ptr++; \ if (size & 0x80) { \ ptr = fastdecode_longsize(ptr, &size); \ } \ \ if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \ &d->input, ptr, size) || \ (size % valbytes) != 0)) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ } \ \ upb_Array** arr_p = fastdecode_fieldmem(msg, data); \ upb_Array* arr = *arr_p; \ uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ int elems = size / valbytes; \ \ if (UPB_LIKELY(!arr)) { \ *arr_p = arr = \ UPB_PRIVATE(_upb_Array_New)(&d->arena, elems, elem_size_lg2); \ if (!arr) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ } \ } else { \ _upb_Array_ResizeUninitialized(arr, elems, &d->arena); \ } \ \ char* dst = _upb_array_ptr(arr); \ memcpy(dst, ptr, size); \ arr->UPB_PRIVATE(size) = elems; \ \ ptr += size; \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, unpacked, packed) \ if (card == CARD_p) { \ FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, unpacked); \ } else { \ FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ valbytes, card, packed); \ } /* Generate all combinations: * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ #define F(card, valbytes, tagbytes) \ UPB_NOINLINE \ const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \ upb_prf##valbytes##_##tagbytes##bt); \ } #define TYPES(card, tagbytes) \ F(card, 4, tagbytes) \ F(card, 8, tagbytes) #define TAGBYTES(card) \ TYPES(card, 1) \ TYPES(card, 2) TAGBYTES(s) TAGBYTES(o) TAGBYTES(r) TAGBYTES(p) #undef F #undef TYPES #undef TAGBYTES #undef FASTDECODE_UNPACKEDFIXED #undef FASTDECODE_PACKEDFIXED /* string fields **************************************************************/ typedef const char* fastdecode_copystr_func(struct upb_Decoder* d, const char* ptr, upb_Message* msg, const upb_MiniTable* table, uint64_t hasbits, upb_StringView* dst); UPB_NOINLINE static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { upb_StringView* dst = (upb_StringView*)data; if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); } UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); } #define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ ptr++; \ if (size & 0x80) { \ ptr = fastdecode_longsize(ptr, &size); \ } \ \ if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \ dst->size = 0; \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ } \ \ const char* s_ptr = ptr; \ ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \ if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \ dst->data = s_ptr; \ dst->size = size; \ \ if (validate_utf8) { \ data = (uint64_t)dst; \ UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ } else { \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ } UPB_NOINLINE static const char* fastdecode_longstring_utf8(struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { upb_StringView* dst = (upb_StringView*)data; FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true); } UPB_NOINLINE static const char* fastdecode_longstring_noutf8( struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, uint64_t hasbits, uint64_t data) { upb_StringView* dst = (upb_StringView*)data; FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); } UPB_FORCEINLINE static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size, int copy, char* data, size_t data_offset, upb_StringView* dst) { d->arena.head.UPB_PRIVATE(ptr) += copy; dst->data = data + data_offset; UPB_UNPOISON_MEMORY_REGION(data, copy); memcpy(data, ptr, copy); UPB_POISON_MEMORY_REGION(data + data_offset + size, copy - data_offset - size); } #define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ card, validate_utf8) \ upb_StringView* dst; \ fastdecode_arr farr; \ int64_t size; \ size_t arena_has; \ size_t common_has; \ char* buf; \ \ UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \ UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ sizeof(upb_StringView), card); \ \ again: \ if (card == CARD_r) { \ dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ } \ \ size = (uint8_t)ptr[tagbytes]; \ ptr += tagbytes + 1; \ dst->size = size; \ \ buf = d->arena.head.UPB_PRIVATE(ptr); \ arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \ common_has = UPB_MIN(arena_has, \ upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \ \ if (UPB_LIKELY(size <= 15 - tagbytes)) { \ if (arena_has < 16) goto longstr; \ fastdecode_docopy(d, ptr - tagbytes - 1, size, 16, buf, tagbytes + 1, \ dst); \ } else if (UPB_LIKELY(size <= 32)) { \ if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ fastdecode_docopy(d, ptr, size, 32, buf, 0, dst); \ } else if (UPB_LIKELY(size <= 64)) { \ if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ fastdecode_docopy(d, ptr, size, 64, buf, 0, dst); \ } else if (UPB_LIKELY(size < 128)) { \ if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ fastdecode_docopy(d, ptr, size, 128, buf, 0, dst); \ } else { \ goto longstr; \ } \ \ ptr += size; \ \ if (card == CARD_r) { \ if (validate_utf8 && \ !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ } \ fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ switch (ret.next) { \ case FD_NEXT_SAMEFIELD: \ dst = ret.dst; \ goto again; \ case FD_NEXT_OTHERFIELD: \ data = ret.tag; \ UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ case FD_NEXT_ATLIMIT: \ return ptr; \ } \ } \ \ if (card != CARD_r && validate_utf8) { \ data = (uint64_t)dst; \ UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ } \ \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ \ longstr: \ if (card == CARD_r) { \ fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \ } \ ptr--; \ if (validate_utf8) { \ UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ hasbits, (uint64_t)dst); \ } else { \ UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ hasbits, (uint64_t)dst); \ } #define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ copyfunc, validate_utf8) \ upb_StringView* dst; \ fastdecode_arr farr; \ int64_t size; \ \ if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ RETURN_GENERIC("string field tag mismatch\n"); \ } \ \ if (UPB_UNLIKELY( \ !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \ UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ } \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ sizeof(upb_StringView), card); \ \ again: \ if (card == CARD_r) { \ dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \ } \ \ size = (int8_t)ptr[tagbytes]; \ ptr += tagbytes + 1; \ \ if (UPB_UNLIKELY( \ !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \ ptr--; \ if (validate_utf8) { \ return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ (uint64_t)dst); \ } else { \ return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ (uint64_t)dst); \ } \ } \ \ dst->data = ptr; \ dst->size = size; \ ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \ dst->size); \ \ if (card == CARD_r) { \ if (validate_utf8 && \ !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \ } \ fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \ switch (ret.next) { \ case FD_NEXT_SAMEFIELD: \ dst = ret.dst; \ goto again; \ case FD_NEXT_OTHERFIELD: \ data = ret.tag; \ UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ case FD_NEXT_ATLIMIT: \ return ptr; \ } \ } \ \ if (card != CARD_r && validate_utf8) { \ data = (uint64_t)dst; \ UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ } \ \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); /* Generate all combinations: * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */ #define s_VALIDATE true #define b_VALIDATE false #define F(card, tagbytes, type) \ UPB_NOINLINE \ const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ CARD_##card, type##_VALIDATE); \ } \ const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \ CARD_##card, upb_c##card##type##_##tagbytes##bt, \ type##_VALIDATE); \ } #define UTF8(card, tagbytes) \ F(card, tagbytes, s) \ F(card, tagbytes, b) #define TAGBYTES(card) \ UTF8(card, 1) \ UTF8(card, 2) TAGBYTES(s) TAGBYTES(o) TAGBYTES(r) #undef s_VALIDATE #undef b_VALIDATE #undef F #undef TAGBYTES #undef FASTDECODE_LONGSTRING #undef FASTDECODE_COPYSTRING #undef FASTDECODE_STRING /* message fields *************************************************************/ UPB_INLINE upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* m, int msg_ceil_bytes) { size_t size = m->UPB_PRIVATE(size) + sizeof(upb_Message_Internal); char* msg_data; if (UPB_LIKELY(msg_ceil_bytes > 0 && UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) { UPB_ASSERT(size <= (size_t)msg_ceil_bytes); msg_data = d->arena.head.UPB_PRIVATE(ptr); d->arena.head.UPB_PRIVATE(ptr) += size; UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes); memset(msg_data, 0, msg_ceil_bytes); UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size); } else { msg_data = (char*)upb_Arena_Malloc(&d->arena, size); memset(msg_data, 0, size); } return msg_data + sizeof(upb_Message_Internal); } typedef struct { intptr_t table; upb_Message* msg; } fastdecode_submsgdata; UPB_FORCEINLINE static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, const char* ptr, void* ctx) { upb_Decoder* d = (upb_Decoder*)e; fastdecode_submsgdata* submsg = ctx; ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); UPB_ASSUME(ptr != NULL); return ptr; } #define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ msg_ceil_bytes, card) \ \ if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ RETURN_GENERIC("submessage field tag mismatch\n"); \ } \ \ if (--d->depth == 0) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \ } \ \ upb_Message** dst; \ uint32_t submsg_idx = (data >> 16) & 0xff; \ const upb_MiniTable* tablep = decode_totablep(table); \ const upb_MiniTable* subtablep = upb_MiniTableSub_Message( \ *UPB_PRIVATE(_upb_MiniTable_GetSubByIndex)(tablep, submsg_idx)); \ fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \ fastdecode_arr farr; \ \ if (subtablep->UPB_PRIVATE(table_mask) == (uint8_t)-1) { \ d->depth++; \ RETURN_GENERIC("submessage doesn't have fast tables."); \ } \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ sizeof(upb_Message*), card); \ \ if (card == CARD_s) { \ *(uint32_t*)msg |= hasbits; \ hasbits = 0; \ } \ \ again: \ if (card == CARD_r) { \ dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \ } \ \ submsg.msg = *dst; \ \ if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \ *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \ } \ \ ptr += tagbytes; \ ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \ \ if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \ _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \ } \ \ if (card == CARD_r) { \ fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \ switch (ret.next) { \ case FD_NEXT_SAMEFIELD: \ dst = ret.dst; \ goto again; \ case FD_NEXT_OTHERFIELD: \ d->depth++; \ data = ret.tag; \ UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \ case FD_NEXT_ATLIMIT: \ d->depth++; \ return ptr; \ } \ } \ \ d->depth++; \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define F(card, tagbytes, size_ceil, ceil_arg) \ const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ UPB_PARSE_PARAMS) { \ FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \ CARD_##card); \ } #define SIZES(card, tagbytes) \ F(card, tagbytes, 64, 64) \ F(card, tagbytes, 128, 128) \ F(card, tagbytes, 192, 192) \ F(card, tagbytes, 256, 256) \ F(card, tagbytes, max, -1) #define TAGBYTES(card) \ SIZES(card, 1) \ SIZES(card, 2) TAGBYTES(s) TAGBYTES(o) TAGBYTES(r) #undef TAGBYTES #undef SIZES #undef F #undef FASTDECODE_SUBMSG #endif /* UPB_FASTTABLE */