From 4f1e48ecc62a0073238bade8945e33173f39e0bf Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Fri, 16 Apr 2021 14:39:29 -0700 Subject: [PATCH 1/9] Updated amalgamator to avoid duplicating license blocks. --- tools/amalgamate.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/amalgamate.py b/tools/amalgamate.py index 97a4f4935c..724dac466f 100755 --- a/tools/amalgamate.py +++ b/tools/amalgamate.py @@ -47,7 +47,6 @@ class Amalgamator: self.output_c.write(open("upb/port_def.inc").read()) self.output_h.write("/* Amalgamated source file */\n") - self.output_h.write('#include ') self.output_h.write(open("upb/port_def.inc").read()) def add_include_path(self, path): @@ -69,7 +68,17 @@ class Amalgamator: if not file: raise RuntimeError("Couldn't open file " + infile_name) - for line in file: + lines = file.readlines() + + has_copyright = lines[1].startswith(" * Copyright") + if has_copyright: + while not lines[0].startswith(" */"): + lines.pop(0) + lines.pop(0) + + lines.insert(0, "\n/** " + infile_name + " " + ("*" * 60) +"/"); + + for line in lines: if not self._process_include(line, outfile): outfile.write(line) From 6e10b5789dc2c98e42ed0878897c96ed081248e2 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Fri, 16 Apr 2021 17:54:45 -0700 Subject: [PATCH 2/9] Removed redundant license from port_def in amalgmation. --- tools/amalgamate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/amalgamate.py b/tools/amalgamate.py index 724dac466f..103b11c351 100755 --- a/tools/amalgamate.py +++ b/tools/amalgamate.py @@ -53,8 +53,8 @@ class Amalgamator: self.include_paths.append(path) def finish(self): - self.output_c.write(open("upb/port_undef.inc").read()) - self.output_h.write(open("upb/port_undef.inc").read()) + self._add_header("upb/port_undef.inc") + self.add_src("upb/port_undef.inc") def _process_file(self, infile_name, outfile): file = None From a4b35aa3889e739fa50d586038171efcaa6c8dc7 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 17 Apr 2021 09:44:52 -0700 Subject: [PATCH 3/9] Everything passes except 4 conformance tests. --- upb/decode_fast.c | 999 +++++++++++++++++++++--------------------- upb/decode_internal.h | 7 +- upb/port_def.inc | 6 +- upb/port_undef.inc | 1 + 4 files changed, 508 insertions(+), 505 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 256c9f93d2..53b512ca80 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -52,8 +52,10 @@ #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data +#include #define RETURN_GENERIC(m) \ - /* fprintf(stderr, m); */ \ + /*fprintf(stderr, m); */ \ + /*__builtin_trap(); */ \ return fastdecode_generic(d, ptr, msg, table, hasbits, 0); typedef enum { @@ -66,19 +68,19 @@ typedef enum { UPB_NOINLINE static const char *fastdecode_isdonefallback(upb_decstate *d, const char *ptr, upb_msg *msg, intptr_t table, - uint64_t hasbits, int overrun) { + uint64_t hasbits, uint64_t data) { + int overrun = data; ptr = decode_isdonefallback_inl(d, ptr, overrun); if (ptr == NULL) { return fastdecode_err(d); } - uint16_t tag = fastdecode_loadtag(ptr); - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, tag); + data = fastdecode_loadtag(ptr); + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); } -UPB_FORCEINLINE -static const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, - upb_msg *msg, intptr_t table, - uint64_t hasbits) { +//UPB_FORCEINLINE +UPB_NOINLINE +static const char *fastdecode_dispatch(UPB_PARSE_PARAMS) { if (UPB_UNLIKELY(ptr >= d->limit_ptr)) { int overrun = ptr - d->end; if (UPB_LIKELY(overrun == d->limit)) { @@ -86,20 +88,27 @@ static const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, *(uint32_t*)msg |= hasbits; // Sync hasbits. return ptr; } else { - return fastdecode_isdonefallback(d, ptr, msg, table, hasbits, overrun); + data = overrun; + UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS); } } // Read two bytes of tag data (for a one-byte tag, the high byte is junk). - uint16_t tag = fastdecode_loadtag(ptr); - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, tag); + data = fastdecode_loadtag(ptr); + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); } UPB_FORCEINLINE -static bool fastdecode_checktag(uint64_t data, int tagbytes) { +static bool fastdecode_checktag(uint16_t data, int tagbytes) { if (tagbytes == 1) { + if (data & 0xff) { + //fprintf(stderr, "YO0: %x\n", (int)data); + } return (data & 0xff) == 0; } else { + if (data & 0xffff) { + //fprintf(stderr, "YO1: %x\n", (int)data); + } return (data & 0xffff) == 0; } } @@ -366,57 +375,54 @@ done: return ptr; } -UPB_FORCEINLINE -static const char *fastdecode_unpackedvarint(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, upb_card card, - bool zigzag, - _upb_field_parser *packed) { - uint64_t val; - void *dst; - fastdecode_arr farr; - - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { - return packed(UPB_PARSE_ARGS); - } - RETURN_GENERIC("varint field tag mismatch\n"); - } - - dst = - fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, card); - if (card == CARD_r) { - if (UPB_UNLIKELY(!dst)) { - RETURN_GENERIC("need array resize\n"); - } - } - -again: - if (card == CARD_r) { - dst = fastdecode_resizearr(d, dst, &farr, valbytes); - } - - ptr += tagbytes; - ptr = fastdecode_varint64(ptr, &val); - if (ptr == NULL) return fastdecode_err(d); - val = fastdecode_munge(val, valbytes, zigzag); - memcpy(dst, &val, valbytes); - - if (card == CARD_r) { - fastdecode_nextret ret = - fastdecode_nextrepeated(d, dst, &ptr, &farr, data, tagbytes, valbytes); - switch (ret.next) { - case FD_NEXT_SAMEFIELD: - dst = ret.dst; - goto again; - case FD_NEXT_OTHERFIELD: - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - case FD_NEXT_ATLIMIT: - return ptr; - } - } - - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} +#define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, packed) \ + uint64_t val; \ + void *dst; \ + fastdecode_arr farr; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return packed(UPB_PARSE_ARGS); \ + } \ + RETURN_GENERIC("varint field tag mismatch\n"); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ + card); \ + if (card == CARD_r) { \ + if (UPB_UNLIKELY(!dst)) { \ + RETURN_GENERIC("need array resize\n"); \ + } \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_varint64(ptr, &val); \ + if (ptr == NULL) return fastdecode_err(d); \ + val = fastdecode_munge(val, valbytes, zigzag); \ + memcpy(dst, &val, valbytes); \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, valbytes); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); typedef struct { uint8_t valbytes; @@ -445,49 +451,42 @@ static const char *fastdecode_topackedvarint(upb_decstate *d, const char *ptr, return ptr; } -UPB_FORCEINLINE -static const char *fastdecode_packedvarint(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, bool zigzag, - _upb_field_parser *unpacked) { - fastdecode_varintdata ctx = {valbytes, zigzag}; - - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - if (fastdecode_flippacked(&data, tagbytes)) { - return unpacked(UPB_PARSE_ARGS); - } else { - RETURN_GENERIC("varint field tag mismatch\n"); - } - } - - ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, - valbytes, CARD_r); - if (UPB_UNLIKELY(!ctx.dst)) { - RETURN_GENERIC("need array resize\n"); - } - - ptr += tagbytes; - ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); - - if (UPB_UNLIKELY(ptr == NULL)) { - return fastdecode_err(d); - } - - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} - -UPB_FORCEINLINE -static const char *fastdecode_varint(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, upb_card card, bool zigzag, - _upb_field_parser *unpacked, - _upb_field_parser *packed) { - if (card == CARD_p) { - return fastdecode_packedvarint(UPB_PARSE_ARGS, tagbytes, valbytes, zigzag, - unpacked); - } else { - return fastdecode_unpackedvarint(UPB_PARSE_ARGS, tagbytes, valbytes, card, - zigzag, packed); +#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, zigzag, unpacked) \ + fastdecode_varintdata ctx = {valbytes, zigzag}; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return unpacked(UPB_PARSE_ARGS); \ + } else { \ + RETURN_GENERIC("varint field tag mismatch\n"); \ + } \ + } \ + \ + ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ + valbytes, CARD_r); \ + if (UPB_UNLIKELY(!ctx.dst)) { \ + RETURN_GENERIC("need array resize\n"); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ + \ + if (UPB_UNLIKELY(ptr == NULL)) { \ + return fastdecode_err(d); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); + +#define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, unpacked, packed) \ + if (card == CARD_p) { \ + FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, zigzag, unpacked); \ + } else { \ + FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, zigzag, packed); \ } -} #define z_ZZ true #define b_ZZ false @@ -499,10 +498,10 @@ static const char *fastdecode_varint(UPB_PARSE_PARAMS, int tagbytes, #define F(card, type, valbytes, tagbytes) \ UPB_NOINLINE \ const char *upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - return fastdecode_varint(UPB_PARSE_ARGS, tagbytes, valbytes, CARD_##card, \ - type##_ZZ, \ - &upb_pr##type##valbytes##_##tagbytes##bt, \ - &upb_pp##type##valbytes##_##tagbytes##bt); \ + FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ + CARD_##card, type##_ZZ, \ + upb_pr##type##valbytes##_##tagbytes##bt, \ + upb_pp##type##valbytes##_##tagbytes##bt); \ } #define TYPES(card, tagbytes) \ @@ -530,126 +529,120 @@ TAGBYTES(p) #undef F #undef TYPES #undef TAGBYTES +#undef FASTDECODE_UNPACKEDVARINT +#undef FASTDECODE_PACKEDVARINT +#undef FASTDECODE_VARINT /* fixed fields ***************************************************************/ -UPB_FORCEINLINE -static const char *fastdecode_unpackedfixed(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, upb_card card, - _upb_field_parser *packed) { - void *dst; - fastdecode_arr farr; - - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { - return packed(UPB_PARSE_ARGS); - } - RETURN_GENERIC("fixed field tag mismatch\n"); - } - - dst = - fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, card); - if (card == CARD_r) { - if (UPB_UNLIKELY(!dst)) { - RETURN_GENERIC("couldn't allocate array in arena\n"); - } +#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, packed) \ + void *dst; \ + fastdecode_arr farr; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return packed(UPB_PARSE_ARGS); \ + } \ + RETURN_GENERIC("fixed field tag mismatch\n"); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ + card); \ + if (card == CARD_r) { \ + if (UPB_UNLIKELY(!dst)) { \ + RETURN_GENERIC("couldn't allocate array in arena\n"); \ + } \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ + } \ + \ + ptr += tagbytes; \ + memcpy(dst, ptr, valbytes); \ + ptr += valbytes; \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, valbytes); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked) \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return unpacked(UPB_PARSE_ARGS); \ + } else { \ + RETURN_GENERIC("varint field tag mismatch\n"); \ + } \ + } \ + \ + ptr += tagbytes; \ + int size = (uint8_t)ptr[0]; \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) || \ + (size % valbytes) != 0) { \ + return fastdecode_err(d); \ + } \ + \ + upb_array **arr_p = fastdecode_fieldmem(msg, data); \ + upb_array *arr = *arr_p; \ + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ + int elems = size / valbytes; \ + \ + if (UPB_LIKELY(!arr)) { \ + *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); \ + if (!arr) { \ + return fastdecode_err(d); \ + } \ + } else { \ + _upb_array_resize(arr, elems, &d->arena); \ + } \ + \ + char *dst = _upb_array_ptr(arr); \ + memcpy(dst, ptr, size); \ + arr->len = elems; \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, unpacked, packed) \ + if (card == CARD_p) { \ + FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked); \ + } else { \ + FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, packed); \ } - -again: - if (card == CARD_r) { - dst = fastdecode_resizearr(d, dst, &farr, valbytes); - } - - ptr += tagbytes; - memcpy(dst, ptr, valbytes); - ptr += valbytes; - - if (card == CARD_r) { - fastdecode_nextret ret = - fastdecode_nextrepeated(d, dst, &ptr, &farr, data, tagbytes, valbytes); - switch (ret.next) { - case FD_NEXT_SAMEFIELD: - dst = ret.dst; - goto again; - case FD_NEXT_OTHERFIELD: - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - case FD_NEXT_ATLIMIT: - return ptr; - } - } - - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} - -UPB_FORCEINLINE -static const char *fastdecode_packedfixed(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, - _upb_field_parser *unpacked) { - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - if (fastdecode_flippacked(&data, tagbytes)) { - return unpacked(UPB_PARSE_ARGS); - } else { - RETURN_GENERIC("varint field tag mismatch\n"); - } - } - - ptr += tagbytes; - int size = (uint8_t)ptr[0]; - ptr++; - if (size & 0x80) { - ptr = fastdecode_longsize(ptr, &size); - } - - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) || - (size % valbytes) != 0) { - return fastdecode_err(d); - } - - upb_array **arr_p = fastdecode_fieldmem(msg, data); - upb_array *arr = *arr_p; - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); - int elems = size / valbytes; - - if (UPB_LIKELY(!arr)) { - *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); - if (!arr) { - return fastdecode_err(d); - } - } else { - _upb_array_resize(arr, elems, &d->arena); - } - - char *dst = _upb_array_ptr(arr); - memcpy(dst, ptr, size); - arr->len = elems; - - return fastdecode_dispatch(d, ptr + size, msg, table, hasbits); -} - -UPB_FORCEINLINE -static const char *fastdecode_fixed(UPB_PARSE_PARAMS, int tagbytes, - int valbytes, upb_card card, - _upb_field_parser *unpacked, - _upb_field_parser *packed) { - if (card == CARD_p) { - return fastdecode_packedfixed(UPB_PARSE_ARGS, tagbytes, valbytes, unpacked); - } else { - return fastdecode_unpackedfixed(UPB_PARSE_ARGS, tagbytes, valbytes, card, - packed); - } -} - /* Generate all combinations: * {s,o,r,p} x {f4,f8} x {1bt,2bt} */ -#define F(card, valbytes, tagbytes) \ - UPB_NOINLINE \ - const char *upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - return fastdecode_fixed(UPB_PARSE_ARGS, tagbytes, valbytes, CARD_##card, \ - &upb_ppf##valbytes##_##tagbytes##bt, \ - &upb_prf##valbytes##_##tagbytes##bt); \ +#define F(card, valbytes, tagbytes) \ + UPB_NOINLINE \ + const char *upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \ + CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \ + upb_prf##valbytes##_##tagbytes##bt); \ } #define TYPES(card, tagbytes) \ @@ -668,6 +661,8 @@ TAGBYTES(p) #undef F #undef TYPES #undef TAGBYTES +#undef FASTDECODE_UNPACKEDFIXED +#undef FASTDECODE_PACKEDFIXED /* string fields **************************************************************/ @@ -676,59 +671,51 @@ typedef const char *fastdecode_copystr_func(struct upb_decstate *d, const upb_msglayout *table, uint64_t hasbits, upb_strview *dst); -UPB_NOINLINE -static const char *fastdecode_verifyutf8(upb_decstate *d, const char *ptr, - upb_msg *msg, intptr_t table, - uint64_t hasbits, upb_strview *dst) { - if (!decode_verifyutf8_inl(dst->data, dst->size)) { - return fastdecode_err(d); - } - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} - -UPB_FORCEINLINE -static const char *fastdecode_longstring(struct upb_decstate *d, - const char *ptr, upb_msg *msg, - intptr_t table, uint64_t hasbits, - upb_strview *dst, - bool validate_utf8) { - int size = (uint8_t)ptr[0]; // Could plumb through hasbits. - ptr++; - if (size & 0x80) { - ptr = fastdecode_longsize(ptr, &size); - } - - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr))) { - dst->size = 0; - return fastdecode_err(d); +#define FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst) \ + if (!decode_verifyutf8_inl(dst->data, dst->size)) { \ + return fastdecode_err(d); \ + } \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ + int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr))) { \ + dst->size = 0; \ + return fastdecode_err(d); \ + } \ + \ + if (d->alias) { \ + dst->data = ptr; \ + dst->size = size; \ + } else { \ + char *data = upb_arena_malloc(&d->arena, size); \ + if (!data) { \ + return fastdecode_err(d); \ + } \ + memcpy(data, ptr, size); \ + dst->data = data; \ + dst->size = size; \ + } \ + \ + ptr += size; \ + if (validate_utf8) { \ + FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ + } else { \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ } - if (d->alias) { - dst->data = ptr; - dst->size = size; - } else { - char *data = upb_arena_malloc(&d->arena, size); - if (!data) { - return fastdecode_err(d); - } - memcpy(data, ptr, size); - dst->data = data; - dst->size = size; - } - - if (validate_utf8) { - return fastdecode_verifyutf8(d, ptr + size, msg, table, hasbits, dst); - } else { - return fastdecode_dispatch(d, ptr + size, msg, table, hasbits); - } -} - UPB_NOINLINE static const char *fastdecode_longstring_utf8(struct upb_decstate *d, - const char *ptr, upb_msg *msg, - intptr_t table, uint64_t hasbits, - upb_strview *dst) { - return fastdecode_longstring(d, ptr, msg, table, hasbits, dst, true); + const char *ptr, upb_msg *msg, + intptr_t table, uint64_t hasbits, + uint64_t data) { + upb_strview *dst = (upb_strview*)data; + FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true); } UPB_NOINLINE @@ -736,8 +723,9 @@ static const char *fastdecode_longstring_noutf8(struct upb_decstate *d, const char *ptr, upb_msg *msg, intptr_t table, uint64_t hasbits, - upb_strview *dst) { - return fastdecode_longstring(d, ptr, msg, table, hasbits, dst, false); + uint64_t data) { + upb_strview *dst = (upb_strview*)data; + FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false); } UPB_FORCEINLINE @@ -750,156 +738,159 @@ static void fastdecode_docopy(upb_decstate *d, const char *ptr, uint32_t size, UPB_POISON_MEMORY_REGION(data + size, copy - size); } -UPB_FORCEINLINE -static const char *fastdecode_copystring(UPB_PARSE_PARAMS, int tagbytes, - upb_card card, bool validate_utf8) { - upb_strview *dst; - fastdecode_arr farr; - int64_t size; - size_t arena_has; - size_t common_has; - char *buf; - - UPB_ASSERT(!d->alias); - UPB_ASSERT(fastdecode_checktag(data, tagbytes)); - - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, - sizeof(upb_strview), card); - -again: - if (card == CARD_r) { - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); - } - - size = (uint8_t)ptr[tagbytes]; - ptr += tagbytes + 1; - dst->size = size; - - buf = d->arena.head.ptr; - arena_has = _upb_arenahas(&d->arena); - common_has = UPB_MIN(arena_has, (d->end - ptr) + 16); - - if (UPB_LIKELY(size <= 15 - tagbytes)) { - if (arena_has < 16) goto longstr; - d->arena.head.ptr += 16; - memcpy(buf, ptr - tagbytes - 1, 16); - dst->data = buf + tagbytes + 1; - } else if (UPB_LIKELY(size <= 32)) { - if (UPB_UNLIKELY(common_has < 32)) goto longstr; - fastdecode_docopy(d, ptr, size, 32, buf, dst); - } else if (UPB_LIKELY(size <= 64)) { - if (UPB_UNLIKELY(common_has < 64)) goto longstr; - fastdecode_docopy(d, ptr, size, 64, buf, dst); - } else if (UPB_LIKELY(size < 128)) { - if (UPB_UNLIKELY(common_has < 128)) goto longstr; - fastdecode_docopy(d, ptr, size, 128, buf, dst); - } else { - goto longstr; - } - - ptr += size; - - if (card == CARD_r) { - if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { - return fastdecode_err(d); - } - fastdecode_nextret ret = fastdecode_nextrepeated( - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); - switch (ret.next) { - case FD_NEXT_SAMEFIELD: - dst = ret.dst; - goto again; - case FD_NEXT_OTHERFIELD: - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - case FD_NEXT_ATLIMIT: - return ptr; - } - } - - if (card != CARD_r && validate_utf8) { - return fastdecode_verifyutf8(d, ptr, msg, table, hasbits, dst); - } - - return fastdecode_dispatch(d, ptr, msg, table, hasbits); - -longstr: - ptr--; - if (validate_utf8) { - return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, dst); - } else { - return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, dst); - } -} - -UPB_FORCEINLINE -static const char *fastdecode_string(UPB_PARSE_PARAMS, int tagbytes, - upb_card card, _upb_field_parser *copyfunc, - bool validate_utf8) { - upb_strview *dst; - fastdecode_arr farr; - int64_t size; - - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - RETURN_GENERIC("string field tag mismatch\n"); - } - - if (UPB_UNLIKELY(!d->alias)) { - return copyfunc(UPB_PARSE_ARGS); - } - - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, - sizeof(upb_strview), card); - -again: - if (card == CARD_r) { - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); +#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + card, validate_utf8) \ + upb_strview *dst; \ + fastdecode_arr farr; \ + int64_t size; \ + size_t arena_has; \ + size_t common_has; \ + char *buf; \ + \ + UPB_ASSERT(!d->alias); \ + UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_strview), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ + } \ + \ + size = (uint8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + dst->size = size; \ + \ + buf = d->arena.head.ptr; \ + arena_has = _upb_arenahas(&d->arena); \ + common_has = UPB_MIN(arena_has, (d->end - ptr) + 16); \ + \ + if (UPB_LIKELY(size <= 15 - tagbytes)) { \ + if (arena_has < 16) goto longstr; \ + d->arena.head.ptr += 16; \ + memcpy(buf, ptr - tagbytes - 1, 16); \ + dst->data = buf + tagbytes + 1; \ + } else if (UPB_LIKELY(size <= 32)) { \ + if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 32, buf, dst); \ + } else if (UPB_LIKELY(size <= 64)) { \ + if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 64, buf, dst); \ + } else if (UPB_LIKELY(size < 128)) { \ + if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ + fastdecode_docopy(d, ptr, size, 128, buf, dst); \ + } else { \ + goto longstr; \ + } \ + \ + ptr += size; \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ + return fastdecode_err(d); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + /* return */ FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ + \ + longstr: \ + ptr--; \ + if (validate_utf8) { \ + UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ + } else { \ + UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ } - size = (int8_t)ptr[tagbytes]; - ptr += tagbytes + 1; - dst->data = ptr; - dst->size = size; - - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) { - ptr--; - if (validate_utf8) { - return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, dst); - } else { - return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, dst); - } - } - - ptr += size; - - if (card == CARD_r) { - if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { - return fastdecode_err(d); - } - fastdecode_nextret ret = fastdecode_nextrepeated( - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); - switch (ret.next) { - case FD_NEXT_SAMEFIELD: - dst = ret.dst; - if (UPB_UNLIKELY(!d->alias)) { - // Buffer flipped and we can't alias any more. Bounce to copyfunc(), - // but via dispatch since we need to reload table data also. - fastdecode_commitarr(dst, &farr, sizeof(upb_strview)); - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - } - goto again; - case FD_NEXT_OTHERFIELD: - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - case FD_NEXT_ATLIMIT: - return ptr; - } - } - - if (card != CARD_r && validate_utf8) { - return fastdecode_verifyutf8(d, ptr, msg, table, hasbits, dst); - } - - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} +#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ + copyfunc, validate_utf8) \ + upb_strview *dst; \ + fastdecode_arr farr; \ + int64_t size; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + RETURN_GENERIC("string field tag mismatch\n"); \ + } \ + \ + if (UPB_UNLIKELY(!d->alias)) { \ + UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_strview), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ + } \ + \ + size = (int8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + dst->data = ptr; \ + dst->size = size; \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) { \ + ptr--; \ + if (validate_utf8) { \ + return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } else { \ + return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } \ + } \ + \ + ptr += size; \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ + return fastdecode_err(d); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + if (UPB_UNLIKELY(!d->alias)) { \ + /* Buffer flipped and we can't alias any more. Bounce to */ \ + /* copyfunc(), but via dispatch since we need to reload table */ \ + /* data also. */ \ + fastdecode_commitarr(dst, &farr, sizeof(upb_strview)); \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + } \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + /* return */ FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); /* Generate all combinations: * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */ @@ -907,16 +898,16 @@ again: #define s_VALIDATE true #define b_VALIDATE false -#define F(card, tagbytes, type) \ - UPB_NOINLINE \ - const char *upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - return fastdecode_copystring(UPB_PARSE_ARGS, tagbytes, CARD_##card, \ - type##_VALIDATE); \ - } \ - const char *upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ - return fastdecode_string(UPB_PARSE_ARGS, tagbytes, CARD_##card, \ - &upb_c##card##type##_##tagbytes##bt, \ - type##_VALIDATE); \ +#define F(card, tagbytes, type) \ + UPB_NOINLINE \ + const char *upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + CARD_##card, type##_VALIDATE); \ + } \ + const char *upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \ + FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + CARD_##card, upb_c##card##type##_##tagbytes##bt, \ + type##_VALIDATE); \ } #define UTF8(card, tagbytes) \ @@ -935,6 +926,10 @@ TAGBYTES(r) #undef b_VALIDATE #undef F #undef TAGBYTES +#undef FASTDECODE_VERIFYUTF8 +#undef FASTDECODE_LONGSTRING +#undef FASTDECODE_COPYSTRING +#undef FASTDECODE_STRING /* message fields *************************************************************/ @@ -967,82 +962,83 @@ UPB_FORCEINLINE static const char *fastdecode_tosubmsg(upb_decstate *d, const char *ptr, void *ctx) { fastdecode_submsgdata *submsg = ctx; - ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0); + ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0); UPB_ASSUME(ptr != NULL); return ptr; } -UPB_FORCEINLINE -static const char *fastdecode_submsg(UPB_PARSE_PARAMS, int tagbytes, - int msg_ceil_bytes, upb_card card) { - - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { - RETURN_GENERIC("submessage field tag mismatch\n"); - } - - if (--d->depth == 0) return fastdecode_err(d); - - upb_msg **dst; - uint32_t submsg_idx = (data >> 16) & 0xff; - const upb_msglayout *tablep = decode_totablep(table); - const upb_msglayout *subtablep = tablep->submsgs[submsg_idx]; - fastdecode_submsgdata submsg = {decode_totable(subtablep)}; - fastdecode_arr farr; - - if (subtablep->table_mask == (uint8_t)-1) { - RETURN_GENERIC("submessage doesn't have fast tables."); - } - - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, - sizeof(upb_msg *), card); - - if (card == CARD_s) { - *(uint32_t*)msg |= hasbits; - hasbits = 0; - } - -again: - if (card == CARD_r) { - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_msg*)); - } - - submsg.msg = *dst; - - if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { - *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); - } - - ptr += tagbytes; - ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); - - if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { - return fastdecode_err(d); - } - - if (card == CARD_r) { - fastdecode_nextret ret = fastdecode_nextrepeated( - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_msg *)); - switch (ret.next) { - case FD_NEXT_SAMEFIELD: - dst = ret.dst; - goto again; - case FD_NEXT_OTHERFIELD: - d->depth++; - return fastdecode_tagdispatch(d, ptr, msg, table, hasbits, ret.tag); - case FD_NEXT_ATLIMIT: - d->depth++; - return ptr; - } - } - - d->depth++; - return fastdecode_dispatch(d, ptr, msg, table, hasbits); -} - -#define F(card, tagbytes, size_ceil, ceil_arg) \ - const char *upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ - UPB_PARSE_PARAMS) { \ - return fastdecode_submsg(UPB_PARSE_ARGS, tagbytes, ceil_arg, CARD_##card); \ +#include +#define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ + msg_ceil_bytes, card) \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + RETURN_GENERIC("submessage field tag mismatch\n"); \ + } \ + \ + if (--d->depth == 0) return fastdecode_err(d); \ + \ + upb_msg **dst; \ + uint32_t submsg_idx = (data >> 16) & 0xff; \ + const upb_msglayout *tablep = decode_totablep(table); \ + const upb_msglayout *subtablep = tablep->submsgs[submsg_idx]; \ + fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \ + fastdecode_arr farr; \ + \ + if (subtablep->table_mask == (uint8_t)-1) { \ + RETURN_GENERIC("submessage doesn't have fast tables."); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_msg *), card); \ + \ + if (card == CARD_s) { \ + *(uint32_t *)msg |= hasbits; \ + hasbits = 0; \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_msg *)); \ + } \ + \ + submsg.msg = *dst; \ + \ + if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \ + *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \ + \ + if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \ + return fastdecode_err(d); \ + } \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_msg *)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + d->depth++; \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + d->depth++; \ + return ptr; \ + } \ + } \ + \ + d->depth++; \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); + +#define F(card, tagbytes, size_ceil, ceil_arg) \ + const char *upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \ + UPB_PARSE_PARAMS) { \ + FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \ + CARD_##card); \ } #define SIZES(card, tagbytes) \ @@ -1063,5 +1059,6 @@ TAGBYTES(r) #undef TAGBYTES #undef SIZES #undef F +#undef FASTDECODE_SUBMSG #endif /* UPB_FASTTABLE */ diff --git a/upb/decode_internal.h b/upb/decode_internal.h index d993078487..f7fdc9df3a 100644 --- a/upb/decode_internal.h +++ b/upb/decode_internal.h @@ -142,10 +142,11 @@ bool decode_isdone(upb_decstate *d, const char **ptr) { } } +#include UPB_INLINE const char *fastdecode_tagdispatch(upb_decstate *d, const char *ptr, upb_msg *msg, intptr_t table, - uint64_t hasbits, uint32_t tag) { + uint64_t hasbits, uint64_t tag) { const upb_msglayout *table_p = decode_totablep(table); uint8_t mask = table; uint64_t data; @@ -153,7 +154,9 @@ const char *fastdecode_tagdispatch(upb_decstate *d, const char *ptr, UPB_ASSUME((idx & 7) == 0); idx >>= 3; data = table_p->fasttable[idx].field_data ^ tag; - return table_p->fasttable[idx].field_parser(d, ptr, msg, table, hasbits, data); + //fprintf(stderr, "Dispatch at: %p\n", ptr); + UPB_MUSTTAIL return table_p->fasttable[idx].field_parser(d, ptr, msg, table, + hasbits, data); } UPB_INLINE uint32_t fastdecode_loadtag(const char* ptr) { diff --git a/upb/port_def.inc b/upb/port_def.inc index 56a5937078..a4d966f0ef 100644 --- a/upb/port_def.inc +++ b/upb/port_def.inc @@ -167,8 +167,10 @@ /* Configure whether fasttable is switched on or not. *************************/ -#if (defined(__x86_64__) || defined(__aarch64__)) && defined(__GNUC__) +#if (defined(__x86_64__) || defined(__aarch64__)) && defined(__clang__) && \ + __has_attribute(musttail) #define UPB_FASTTABLE_SUPPORTED 1 +#define UPB_MUSTTAIL __attribute__((musttail)) #else #define UPB_FASTTABLE_SUPPORTED 0 #endif @@ -178,7 +180,7 @@ * for example for testing or benchmarking. */ #if defined(UPB_ENABLE_FASTTABLE) #if !UPB_FASTTABLE_SUPPORTED -#error fasttable is x86-64 + Clang/GCC only +#error fasttable is x86-64/ARM64 only and requires a recent Clang that supports __attribute__((musttail)) #endif #define UPB_FASTTABLE 1 /* Define UPB_TRY_ENABLE_FASTTABLE to use fasttable if possible. diff --git a/upb/port_undef.inc b/upb/port_undef.inc index b96d2993d0..70956df25d 100644 --- a/upb/port_undef.inc +++ b/upb/port_undef.inc @@ -52,6 +52,7 @@ #undef UPB_SETJMP #undef UPB_LONGJMP #undef UPB_PTRADD +#undef UPB_MUSTTAIL #undef UPB_FASTTABLE_SUPPORTED #undef UPB_FASTTABLE #undef UPB_FASTTABLE_INIT From e84793dd73e19528dafa40781e0febfe88ab0e2b Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 17 Apr 2021 10:58:10 -0700 Subject: [PATCH 4/9] Cleaned up debugging artifacts. --- upb/decode_fast.c | 12 +++++------- upb/decode_internal.h | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 53b512ca80..16f0881664 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -52,10 +52,9 @@ #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data -#include -#define RETURN_GENERIC(m) \ - /*fprintf(stderr, m); */ \ - /*__builtin_trap(); */ \ +#define RETURN_GENERIC(m) \ + /* fprintf(stderr, m); */ \ + /*__builtin_trap(); */ \ return fastdecode_generic(d, ptr, msg, table, hasbits, 0); typedef enum { @@ -78,8 +77,7 @@ static const char *fastdecode_isdonefallback(upb_decstate *d, const char *ptr, UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); } -//UPB_FORCEINLINE -UPB_NOINLINE +UPB_FORCEINLINE static const char *fastdecode_dispatch(UPB_PARSE_PARAMS) { if (UPB_UNLIKELY(ptr >= d->limit_ptr)) { int overrun = ptr - d->end; @@ -622,6 +620,7 @@ TAGBYTES(p) memcpy(dst, ptr, size); \ arr->len = elems; \ \ + ptr += size; \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ @@ -967,7 +966,6 @@ static const char *fastdecode_tosubmsg(upb_decstate *d, const char *ptr, return ptr; } -#include #define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \ msg_ceil_bytes, card) \ \ diff --git a/upb/decode_internal.h b/upb/decode_internal.h index f7fdc9df3a..0492363725 100644 --- a/upb/decode_internal.h +++ b/upb/decode_internal.h @@ -142,7 +142,7 @@ bool decode_isdone(upb_decstate *d, const char **ptr) { } } -#include +#if UPB_FASTTABLE UPB_INLINE const char *fastdecode_tagdispatch(upb_decstate *d, const char *ptr, upb_msg *msg, intptr_t table, @@ -154,10 +154,10 @@ const char *fastdecode_tagdispatch(upb_decstate *d, const char *ptr, UPB_ASSUME((idx & 7) == 0); idx >>= 3; data = table_p->fasttable[idx].field_data ^ tag; - //fprintf(stderr, "Dispatch at: %p\n", ptr); UPB_MUSTTAIL return table_p->fasttable[idx].field_parser(d, ptr, msg, table, hasbits, data); } +#endif UPB_INLINE uint32_t fastdecode_loadtag(const char* ptr) { uint16_t tag; From 75df4cdaa3dbe267d903275319e787f6e08eff7e Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 17 Apr 2021 15:40:32 -0700 Subject: [PATCH 5/9] Loosened the check for fasttable to match what we did before. --- upb/port_def.inc | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/upb/port_def.inc b/upb/port_def.inc index a4d966f0ef..0f9a157bea 100644 --- a/upb/port_def.inc +++ b/upb/port_def.inc @@ -167,11 +167,29 @@ /* Configure whether fasttable is switched on or not. *************************/ -#if (defined(__x86_64__) || defined(__aarch64__)) && defined(__clang__) && \ - __has_attribute(musttail) -#define UPB_FASTTABLE_SUPPORTED 1 +#if defined(__clang__) && __has_attribute(musttail) #define UPB_MUSTTAIL __attribute__((musttail)) #else +#define UPB_MUSTTAIL +#endif + +/* This check is not fully robust: it does not require that we have "musttail" + * support available. We need tail calls to avoid consuming arbitrary amounts + * of stack space. + * + * GCC/Clang can mostly be trusted to generate tail calls as long as + * optimization is enabled, but, debug builds will not generate tail calls + * unless "musttail" is available. + * + * We should probably either: + * 1. require that the compiler supports musttail. + * 2. add some fallback code for when musttail isn't available (ie. return + * instead of tail calling). This is safe and portable, but this comes at + * a CPU cost. + */ +#if (defined(__x86_64__) || defined(__aarch64__)) && defined(__GNUC__) +#define UPB_FASTTABLE_SUPPORTED 1 +#else #define UPB_FASTTABLE_SUPPORTED 0 #endif @@ -180,7 +198,7 @@ * for example for testing or benchmarking. */ #if defined(UPB_ENABLE_FASTTABLE) #if !UPB_FASTTABLE_SUPPORTED -#error fasttable is x86-64/ARM64 only and requires a recent Clang that supports __attribute__((musttail)) +#error fasttable is x86-64/ARM64 only and requires GCC or Clang. #endif #define UPB_FASTTABLE 1 /* Define UPB_TRY_ENABLE_FASTTABLE to use fasttable if possible. From 286441afa745a81c1ee710fa86bd6ce96fe8743a Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 17 Apr 2021 16:23:30 -0700 Subject: [PATCH 6/9] Fixed a size regression due to inlining UTF-8 verification. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overall size/speed impact on fasttable decoder is now: name old time/op new time/op delta ArenaOneAlloc 21.5ns ± 0% 21.5ns ± 0% ~ (p=0.060 n=12+12) ArenaInitialBlockOneAlloc 6.33ns ± 0% 6.33ns ± 0% ~ (p=0.413 n=11+12) LoadDescriptor_Upb 43.4µs ± 1% 45.5µs ± 1% +4.79% (p=0.000 n=12+12) LoadAdsDescriptor_Upb 2.50ms ± 0% 2.51ms ± 2% ~ (p=0.512 n=10+11) LoadDescriptor_Proto2 240µs ± 0% 240µs ± 0% -0.25% (p=0.000 n=12+12) LoadAdsDescriptor_Proto2 12.9ms ± 0% 12.9ms ± 0% +0.20% (p=0.014 n=10+12) Parse_Upb_FileDesc 4.99µs ± 0% 5.04µs ± 0% +0.98% (p=0.000 n=11+10) Parse_Upb_FileDesc 4.02µs ± 0% 4.18µs ± 0% +4.16% (p=0.000 n=10+12) Parse_Upb_FileDesc 4.49µs ± 0% 4.54µs ± 0% +1.16% (p=0.000 n=11+10) Parse_Upb_FileDesc 3.60µs ± 0% 3.80µs ± 0% +5.73% (p=0.000 n=12+11) Parse_Proto2 29.3µs ± 0% 29.3µs ± 0% ~ (p=0.069 n=11+12) Parse_Proto2 20.2µs ± 3% 20.3µs ± 2% ~ (p=0.880 n=12+11) Parse_Proto2 16.5µs ± 0% 16.5µs ± 0% ~ (p=1.000 n=12+12) Parse_Proto2 16.4µs ± 0% 16.4µs ± 1% ~ (p=0.590 n=12+12) SerializeDescriptor_Proto2 5.31µs ± 1% 6.65µs ±29% +25.07% (p=0.000 n=12+12) SerializeDescriptor_Upb 12.4µs ± 0% 12.5µs ± 0% +1.23% (p=0.000 n=12+12) FILE SIZE VM SIZE -------------- -------------- +16% +128 [ = ] 0 [Unmapped] -1.2% -4 -1.2% -4 [section .text] [NEW] +2 [NEW] +2 fastdecode_isdonefallback [DEL] -6 [DEL] -6 fastdecode_longstring_noutf8 -0.2% -124 -0.2% -124 upb/decode_fast.c +5.8% +64 +6.0% +64 upb_pom_1bt_max64b +2.7% +64 +2.7% +64 upb_ppv8_2bt +2.7% +32 +2.8% +32 upb_psm_1bt_max256b +2.8% +32 +3.0% +32 upb_psm_1bt_max64b +2.8% +32 +3.0% +32 upb_psm_2bt_max64b +4.0% +24 +4.2% +24 upb_psv8_1bt +2.0% +16 +2.1% +16 upb_prf4_2bt +1.3% +16 +1.4% +16 upb_prz8_2bt -0.3% -4 -0.3% -4 [3 Others] -1.6% -8 -1.7% -8 upb_cob_1bt -1.6% -8 -1.7% -8 upb_csb_1bt -2.5% -16 -2.6% -16 upb_pov4_1bt -1.3% -16 -1.3% -16 upb_prv8_2bt -2.5% -16 -2.7% -16 upb_psv4_1bt -2.5% -16 -2.6% -16 upb_psv4_2bt -3.0% -32 -3.1% -32 upb_prs_2bt -2.6% -32 -2.6% -32 upb_prv4_2bt -4.9% -48 -5.1% -48 upb_prb_2bt -3.9% -48 -4.0% -48 upb_prv4_1bt -7.2% -72 -7.5% -72 upb_prb_1bt -7.8% -88 -8.0% -88 upb_prs_1bt [ = ] 0 -0.1% -128 TOTAL There is a bit of speed regression, but it appears there were bigger CPU regressions prior to this. We probably need some separate optimization attention again to get back to the performance numbers we had when fasttable was first submitted. --- upb/decode_fast.c | 323 ++++++++++++++++++++++++---------------------- 1 file changed, 167 insertions(+), 156 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 16f0881664..974dd139ab 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -670,11 +670,16 @@ typedef const char *fastdecode_copystr_func(struct upb_decstate *d, const upb_msglayout *table, uint64_t hasbits, upb_strview *dst); -#define FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst) \ - if (!decode_verifyutf8_inl(dst->data, dst->size)) { \ - return fastdecode_err(d); \ - } \ +UPB_NOINLINE +static const char *fastdecode_verifyutf8(upb_decstate *d, const char *ptr, + upb_msg *msg, intptr_t table, + uint64_t hasbits, uint64_t data) { + upb_strview *dst = (upb_strview*)data; + if (!decode_verifyutf8_inl(dst->data, dst->size)) { + return fastdecode_err(d); + } UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); +} #define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \ int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \ @@ -703,7 +708,8 @@ typedef const char *fastdecode_copystr_func(struct upb_decstate *d, \ ptr += size; \ if (validate_utf8) { \ - FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ } else { \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ } @@ -737,158 +743,164 @@ static void fastdecode_docopy(upb_decstate *d, const char *ptr, uint32_t size, UPB_POISON_MEMORY_REGION(data + size, copy - size); } -#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ - card, validate_utf8) \ - upb_strview *dst; \ - fastdecode_arr farr; \ - int64_t size; \ - size_t arena_has; \ - size_t common_has; \ - char *buf; \ - \ - UPB_ASSERT(!d->alias); \ - UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ - sizeof(upb_strview), card); \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ - } \ - \ - size = (uint8_t)ptr[tagbytes]; \ - ptr += tagbytes + 1; \ - dst->size = size; \ - \ - buf = d->arena.head.ptr; \ - arena_has = _upb_arenahas(&d->arena); \ - common_has = UPB_MIN(arena_has, (d->end - ptr) + 16); \ - \ - if (UPB_LIKELY(size <= 15 - tagbytes)) { \ - if (arena_has < 16) goto longstr; \ - d->arena.head.ptr += 16; \ - memcpy(buf, ptr - tagbytes - 1, 16); \ - dst->data = buf + tagbytes + 1; \ - } else if (UPB_LIKELY(size <= 32)) { \ - if (UPB_UNLIKELY(common_has < 32)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 32, buf, dst); \ - } else if (UPB_LIKELY(size <= 64)) { \ - if (UPB_UNLIKELY(common_has < 64)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 64, buf, dst); \ - } else if (UPB_LIKELY(size < 128)) { \ - if (UPB_UNLIKELY(common_has < 128)) goto longstr; \ - fastdecode_docopy(d, ptr, size, 128, buf, dst); \ - } else { \ - goto longstr; \ - } \ - \ - ptr += size; \ - \ - if (card == CARD_r) { \ - if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ - return fastdecode_err(d); \ - } \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - if (card != CARD_r && validate_utf8) { \ - /* return */ FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ - } \ - \ - UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ - \ - longstr: \ - ptr--; \ - if (validate_utf8) { \ - UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ - hasbits, (uint64_t)dst); \ - } else { \ - UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ - hasbits, (uint64_t)dst); \ +#define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \ + card, validate_utf8) \ + upb_strview *dst; \ + fastdecode_arr farr; \ + int64_t size; \ + size_t arena_has; \ + size_t common_has; \ + char *buf; \ + \ + UPB_ASSERT(!d->alias); \ + UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_strview), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ + } \ + \ + size = (uint8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + dst->size = size; \ + \ + buf = d->arena.head.ptr; \ + arena_has = _upb_arenahas(&d->arena); \ + common_has = UPB_MIN(arena_has, (d->end - ptr) + 16); \ + \ + if (UPB_LIKELY(size <= 15 - tagbytes)) { \ + if (arena_has < 16) \ + goto longstr; \ + d->arena.head.ptr += 16; \ + memcpy(buf, ptr - tagbytes - 1, 16); \ + dst->data = buf + tagbytes + 1; \ + } else if (UPB_LIKELY(size <= 32)) { \ + if (UPB_UNLIKELY(common_has < 32)) \ + goto longstr; \ + fastdecode_docopy(d, ptr, size, 32, buf, dst); \ + } else if (UPB_LIKELY(size <= 64)) { \ + if (UPB_UNLIKELY(common_has < 64)) \ + goto longstr; \ + fastdecode_docopy(d, ptr, size, 64, buf, dst); \ + } else if (UPB_LIKELY(size < 128)) { \ + if (UPB_UNLIKELY(common_has < 128)) \ + goto longstr; \ + fastdecode_docopy(d, ptr, size, 128, buf, dst); \ + } else { \ + goto longstr; \ + } \ + \ + ptr += size; \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ + return fastdecode_err(d); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ + } \ + \ + UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \ + \ + longstr: \ + ptr--; \ + if (validate_utf8) { \ + UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ + } else { \ + UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \ + hasbits, (uint64_t)dst); \ } -#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ - copyfunc, validate_utf8) \ - upb_strview *dst; \ - fastdecode_arr farr; \ - int64_t size; \ - \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - RETURN_GENERIC("string field tag mismatch\n"); \ - } \ - \ - if (UPB_UNLIKELY(!d->alias)) { \ - UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ - } \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ - sizeof(upb_strview), card); \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ - } \ - \ - size = (int8_t)ptr[tagbytes]; \ - ptr += tagbytes + 1; \ - dst->data = ptr; \ - dst->size = size; \ - \ - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) { \ - ptr--; \ - if (validate_utf8) { \ - return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ - (uint64_t)dst); \ - } else { \ - return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ - (uint64_t)dst); \ - } \ - } \ - \ - ptr += size; \ - \ - if (card == CARD_r) { \ - if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ - return fastdecode_err(d); \ - } \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - if (UPB_UNLIKELY(!d->alias)) { \ - /* Buffer flipped and we can't alias any more. Bounce to */ \ - /* copyfunc(), but via dispatch since we need to reload table */ \ - /* data also. */ \ - fastdecode_commitarr(dst, &farr, sizeof(upb_strview)); \ - data = ret.tag; \ - UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ - } \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ - if (card != CARD_r && validate_utf8) { \ - /* return */ FASTDECODE_VERIFYUTF8(d, ptr, msg, table, hasbits, dst); \ - } \ - \ +#define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \ + copyfunc, validate_utf8) \ + upb_strview *dst; \ + fastdecode_arr farr; \ + int64_t size; \ + \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + RETURN_GENERIC("string field tag mismatch\n"); \ + } \ + \ + if (UPB_UNLIKELY(!d->alias)) { \ + UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \ + } \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \ + sizeof(upb_strview), card); \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_strview)); \ + } \ + \ + size = (int8_t)ptr[tagbytes]; \ + ptr += tagbytes + 1; \ + dst->data = ptr; \ + dst->size = size; \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->end))) { \ + ptr--; \ + if (validate_utf8) { \ + return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } else { \ + return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \ + (uint64_t)dst); \ + } \ + } \ + \ + ptr += size; \ + \ + if (card == CARD_r) { \ + if (validate_utf8 && !decode_verifyutf8_inl(dst->data, dst->size)) { \ + return fastdecode_err(d); \ + } \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_strview)); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + if (UPB_UNLIKELY(!d->alias)) { \ + /* Buffer flipped and we can't alias any more. Bounce to */ \ + /* copyfunc(), but via dispatch since we need to reload table */ \ + /* data also. */ \ + fastdecode_commitarr(dst, &farr, sizeof(upb_strview)); \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + } \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ + if (card != CARD_r && validate_utf8) { \ + data = (uint64_t)dst; \ + UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \ + } \ + \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); /* Generate all combinations: @@ -925,7 +937,6 @@ TAGBYTES(r) #undef b_VALIDATE #undef F #undef TAGBYTES -#undef FASTDECODE_VERIFYUTF8 #undef FASTDECODE_LONGSTRING #undef FASTDECODE_COPYSTRING #undef FASTDECODE_STRING From 876abae2db558af67939b6b25b289b4aeb19f9ec Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 17 Apr 2021 19:38:25 -0700 Subject: [PATCH 7/9] Removed some debug printing and simplified checktag slightly. --- upb/decode_fast.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 974dd139ab..b1d66884d2 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -99,15 +99,9 @@ static const char *fastdecode_dispatch(UPB_PARSE_PARAMS) { UPB_FORCEINLINE static bool fastdecode_checktag(uint16_t data, int tagbytes) { if (tagbytes == 1) { - if (data & 0xff) { - //fprintf(stderr, "YO0: %x\n", (int)data); - } return (data & 0xff) == 0; } else { - if (data & 0xffff) { - //fprintf(stderr, "YO1: %x\n", (int)data); - } - return (data & 0xffff) == 0; + return data == 0; } } From ed708fcd5d0b07528677b63f50a0afd4d0b1d2ca Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sun, 18 Apr 2021 15:46:44 -0700 Subject: [PATCH 8/9] Addressed PR comments. --- upb/decode_fast.c | 248 ++++++++++++++++++++++------------------------ 1 file changed, 117 insertions(+), 131 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index b1d66884d2..2644f94f32 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -52,9 +52,10 @@ #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data -#define RETURN_GENERIC(m) \ - /* fprintf(stderr, m); */ \ - /*__builtin_trap(); */ \ +#define RETURN_GENERIC(m) \ + /* Uncomment either of these for debugging purposes. */ \ + /* fprintf(stderr, m); */ \ + /*__builtin_trap(); */ \ return fastdecode_generic(d, ptr, msg, table, hasbits, 0); typedef enum { @@ -65,9 +66,7 @@ typedef enum { } upb_card; UPB_NOINLINE -static const char *fastdecode_isdonefallback(upb_decstate *d, const char *ptr, - upb_msg *msg, intptr_t table, - uint64_t hasbits, uint64_t data) { +static const char *fastdecode_isdonefallback(UPB_PARSE_PARAMS) { int overrun = data; ptr = decode_isdonefallback_inl(d, ptr, overrun); if (ptr == NULL) { @@ -325,6 +324,14 @@ static bool fastdecode_flippacked(uint64_t *data, int tagbytes) { return fastdecode_checktag(*data, tagbytes); } +#define FASTDECODE_CHECKPACKED(tagbytes, card, func) \ + if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ + if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ + UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \ + } \ + RETURN_GENERIC("packed check tag mismatch\n"); \ + } + /* varint fields **************************************************************/ UPB_FORCEINLINE @@ -373,12 +380,7 @@ done: void *dst; \ fastdecode_arr farr; \ \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ - UPB_MUSTTAIL return packed(UPB_PARSE_ARGS); \ - } \ - RETURN_GENERIC("varint field tag mismatch\n"); \ - } \ + FASTDECODE_CHECKPACKED(tagbytes, card, packed); \ \ dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ card); \ @@ -395,7 +397,8 @@ done: \ ptr += tagbytes; \ ptr = fastdecode_varint64(ptr, &val); \ - if (ptr == NULL) return fastdecode_err(d); \ + if (ptr == NULL) \ + return fastdecode_err(d); \ val = fastdecode_munge(val, valbytes, zigzag); \ memcpy(dst, &val, valbytes); \ \ @@ -403,14 +406,14 @@ done: fastdecode_nextret ret = fastdecode_nextrepeated( \ d, dst, &ptr, &farr, data, tagbytes, valbytes); \ switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ } \ } \ \ @@ -443,31 +446,25 @@ static const char *fastdecode_topackedvarint(upb_decstate *d, const char *ptr, return ptr; } -#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, zigzag, unpacked) \ - fastdecode_varintdata ctx = {valbytes, zigzag}; \ - \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - if (fastdecode_flippacked(&data, tagbytes)) { \ - UPB_MUSTTAIL return unpacked(UPB_PARSE_ARGS); \ - } else { \ - RETURN_GENERIC("varint field tag mismatch\n"); \ - } \ - } \ - \ - ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ - valbytes, CARD_r); \ - if (UPB_UNLIKELY(!ctx.dst)) { \ - RETURN_GENERIC("need array resize\n"); \ - } \ - \ - ptr += tagbytes; \ - ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ - \ - if (UPB_UNLIKELY(ptr == NULL)) { \ - return fastdecode_err(d); \ - } \ - \ +#define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, zigzag, unpacked) \ + fastdecode_varintdata ctx = {valbytes, zigzag}; \ + \ + FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \ + \ + ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \ + valbytes, CARD_r); \ + if (UPB_UNLIKELY(!ctx.dst)) { \ + RETURN_GENERIC("need array resize\n"); \ + } \ + \ + ptr += tagbytes; \ + ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \ + \ + if (UPB_UNLIKELY(ptr == NULL)) { \ + return fastdecode_err(d); \ + } \ + \ UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0); #define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \ @@ -528,93 +525,82 @@ TAGBYTES(p) /* fixed fields ***************************************************************/ -#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, card, packed) \ - void *dst; \ - fastdecode_arr farr; \ - \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \ - UPB_MUSTTAIL return packed(UPB_PARSE_ARGS); \ - } \ - RETURN_GENERIC("fixed field tag mismatch\n"); \ - } \ - \ - dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ - card); \ - if (card == CARD_r) { \ - if (UPB_UNLIKELY(!dst)) { \ - RETURN_GENERIC("couldn't allocate array in arena\n"); \ - } \ - } \ - \ - again: \ - if (card == CARD_r) { \ - dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ - } \ - \ - ptr += tagbytes; \ - memcpy(dst, ptr, valbytes); \ - ptr += valbytes; \ - \ - if (card == CARD_r) { \ - fastdecode_nextret ret = fastdecode_nextrepeated( \ - d, dst, &ptr, &farr, data, tagbytes, valbytes); \ - switch (ret.next) { \ - case FD_NEXT_SAMEFIELD: \ - dst = ret.dst; \ - goto again; \ - case FD_NEXT_OTHERFIELD: \ - data = ret.tag; \ - UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ - case FD_NEXT_ATLIMIT: \ - return ptr; \ - } \ - } \ - \ +#define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, card, packed) \ + void *dst; \ + fastdecode_arr farr; \ + \ + FASTDECODE_CHECKPACKED(tagbytes, card, packed) \ + \ + dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \ + card); \ + if (card == CARD_r) { \ + if (UPB_UNLIKELY(!dst)) { \ + RETURN_GENERIC("couldn't allocate array in arena\n"); \ + } \ + } \ + \ + again: \ + if (card == CARD_r) { \ + dst = fastdecode_resizearr(d, dst, &farr, valbytes); \ + } \ + \ + ptr += tagbytes; \ + memcpy(dst, ptr, valbytes); \ + ptr += valbytes; \ + \ + if (card == CARD_r) { \ + fastdecode_nextret ret = fastdecode_nextrepeated( \ + d, dst, &ptr, &farr, data, tagbytes, valbytes); \ + switch (ret.next) { \ + case FD_NEXT_SAMEFIELD: \ + dst = ret.dst; \ + goto again; \ + case FD_NEXT_OTHERFIELD: \ + data = ret.tag; \ + UPB_MUSTTAIL return fastdecode_tagdispatch(UPB_PARSE_ARGS); \ + case FD_NEXT_ATLIMIT: \ + return ptr; \ + } \ + } \ + \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); -#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, unpacked) \ - if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \ - if (fastdecode_flippacked(&data, tagbytes)) { \ - UPB_MUSTTAIL return unpacked(UPB_PARSE_ARGS); \ - } else { \ - RETURN_GENERIC("varint field tag mismatch\n"); \ - } \ - } \ - \ - ptr += tagbytes; \ - int size = (uint8_t)ptr[0]; \ - ptr++; \ - if (size & 0x80) { \ - ptr = fastdecode_longsize(ptr, &size); \ - } \ - \ - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) || \ - (size % valbytes) != 0) { \ - return fastdecode_err(d); \ - } \ - \ - upb_array **arr_p = fastdecode_fieldmem(msg, data); \ - upb_array *arr = *arr_p; \ - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ - int elems = size / valbytes; \ - \ - if (UPB_LIKELY(!arr)) { \ - *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); \ - if (!arr) { \ - return fastdecode_err(d); \ - } \ - } else { \ - _upb_array_resize(arr, elems, &d->arena); \ - } \ - \ - char *dst = _upb_array_ptr(arr); \ - memcpy(dst, ptr, size); \ - arr->len = elems; \ - \ - ptr += size; \ +#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked) \ + FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ + \ + ptr += tagbytes; \ + int size = (uint8_t)ptr[0]; \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) || \ + (size % valbytes) != 0) { \ + return fastdecode_err(d); \ + } \ + \ + upb_array **arr_p = fastdecode_fieldmem(msg, data); \ + upb_array *arr = *arr_p; \ + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ + int elems = size / valbytes; \ + \ + if (UPB_LIKELY(!arr)) { \ + *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); \ + if (!arr) { \ + return fastdecode_err(d); \ + } \ + } else { \ + _upb_array_resize(arr, elems, &d->arena); \ + } \ + \ + char *dst = _upb_array_ptr(arr); \ + memcpy(dst, ptr, size); \ + arr->len = elems; \ + \ + ptr += size; \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ From 4132034634305f382ffdc7f018782ac81ce99fa5 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 20 Apr 2021 16:08:20 -0700 Subject: [PATCH 9/9] Addressed PR comment. --- upb/decode_fast.c | 70 +++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 2644f94f32..437b647a12 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -566,41 +566,41 @@ TAGBYTES(p) \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); -#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ - valbytes, unpacked) \ - FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ - \ - ptr += tagbytes; \ - int size = (uint8_t)ptr[0]; \ - ptr++; \ - if (size & 0x80) { \ - ptr = fastdecode_longsize(ptr, &size); \ - } \ - \ - if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr)) || \ - (size % valbytes) != 0) { \ - return fastdecode_err(d); \ - } \ - \ - upb_array **arr_p = fastdecode_fieldmem(msg, data); \ - upb_array *arr = *arr_p; \ - uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ - int elems = size / valbytes; \ - \ - if (UPB_LIKELY(!arr)) { \ - *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); \ - if (!arr) { \ - return fastdecode_err(d); \ - } \ - } else { \ - _upb_array_resize(arr, elems, &d->arena); \ - } \ - \ - char *dst = _upb_array_ptr(arr); \ - memcpy(dst, ptr, size); \ - arr->len = elems; \ - \ - ptr += size; \ +#define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \ + valbytes, unpacked) \ + FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \ + \ + ptr += tagbytes; \ + int size = (uint8_t)ptr[0]; \ + ptr++; \ + if (size & 0x80) { \ + ptr = fastdecode_longsize(ptr, &size); \ + } \ + \ + if (UPB_UNLIKELY(fastdecode_boundscheck(ptr, size, d->limit_ptr) || \ + (size % valbytes) != 0)) { \ + return fastdecode_err(d); \ + } \ + \ + upb_array **arr_p = fastdecode_fieldmem(msg, data); \ + upb_array *arr = *arr_p; \ + uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \ + int elems = size / valbytes; \ + \ + if (UPB_LIKELY(!arr)) { \ + *arr_p = arr = _upb_array_new(&d->arena, elems, elem_size_lg2); \ + if (!arr) { \ + return fastdecode_err(d); \ + } \ + } else { \ + _upb_array_resize(arr, elems, &d->arena); \ + } \ + \ + char *dst = _upb_array_ptr(arr); \ + memcpy(dst, ptr, size); \ + arr->len = elems; \ + \ + ptr += size; \ UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \