WIP.

upb/decode.c

@@ -491,101 +491,108 @@ static const char *decode_tomsg(upb_decstate *d, const char *ptr, upb_msg *msg,
   return ptr;
 }
 
-static const char *decode_msg(upb_decstate *d, const char *ptr, upb_msg *msg,
-                              const upb_msglayout *layout) {
-  while (ptr < d->limit) {
-    uint32_t tag;
-    const upb_msglayout_field *field;
-    int field_number;
-    int wire_type;
-    const char *field_start = ptr;
-    wireval val;
-    int op;
-
-    ptr = decode_varint32(d, ptr, d->limit, &tag);
-    field_number = tag >> 3;
-    wire_type = tag & 7;
-
-    field = upb_find_field(layout, field_number);
-
-    switch (wire_type) {
-      case UPB_WIRE_TYPE_VARINT:
-        ptr = decode_varint64(d, ptr, d->limit, &val.uint64_val);
-        op = varint_ops[field->descriptortype];
-        decode_munge(field->descriptortype, &val);
-        break;
-      case UPB_WIRE_TYPE_32BIT:
-        if (d->limit - ptr < 4) decode_err(d);
-        memcpy(&val.uint32_val, ptr, 4);
-        val.uint32_val = _upb_be_swap32(val.uint32_val);
-        ptr += 4;
-        op = OP_SCALAR_LG2(2);
-        if (((1 << field->descriptortype) & fixed32_ok) == 0) goto unknown;
-        break;
-      case UPB_WIRE_TYPE_64BIT:
-        if (d->limit - ptr < 8) decode_err(d);
-        memcpy(&val.uint64_val, ptr, 8);
-        val.uint64_val = _upb_be_swap64(val.uint64_val);
-        ptr += 8;
-        op = OP_SCALAR_LG2(3);
-        if (((1 << field->descriptortype) & fixed64_ok) == 0) goto unknown;
-        break;
-      case UPB_WIRE_TYPE_DELIMITED: {
-        uint32_t size;
-        int ndx = field->descriptortype;
-        if (_upb_isrepeated(field)) ndx += 18;
-        ptr = decode_varint32(d, ptr, d->limit, &size);
-        if (size >= INT32_MAX || (size_t)(d->limit - ptr) < size) {
-          decode_err(d); /* Length overflow. */
-        }
-        val.str_val.data = ptr;
-        val.str_val.size = size;
-        ptr += size;
-        op = delim_ops[ndx];
-        break;
+const char *decode_field(upb_decstate *d, const char *ptr, upb_msg *msg,
+                         const upb_msglayout *layout) {
+  uint32_t tag;
+  const upb_msglayout_field *field;
+  int field_number;
+  int wire_type;
+  const char *field_start = ptr;
+  wireval val;
+  int op;
+
+  ptr = decode_varint32(d, ptr, d->limit, &tag);
+  field_number = tag >> 3;
+  wire_type = tag & 7;
+
+  field = upb_find_field(layout, field_number);
+
+  switch (wire_type) {
+    case UPB_WIRE_TYPE_VARINT:
+      ptr = decode_varint64(d, ptr, d->limit, &val.uint64_val);
+      op = varint_ops[field->descriptortype];
+      decode_munge(field->descriptortype, &val);
+      break;
+    case UPB_WIRE_TYPE_32BIT:
+      if (d->limit - ptr < 4) decode_err(d);
+      memcpy(&val.uint32_val, ptr, 4);
+      val.uint32_val = _upb_be_swap32(val.uint32_val);
+      ptr += 4;
+      op = OP_SCALAR_LG2(2);
+      if (((1 << field->descriptortype) & fixed32_ok) == 0) goto unknown;
+      break;
+    case UPB_WIRE_TYPE_64BIT:
+      if (d->limit - ptr < 8) decode_err(d);
+      memcpy(&val.uint64_val, ptr, 8);
+      val.uint64_val = _upb_be_swap64(val.uint64_val);
+      ptr += 8;
+      op = OP_SCALAR_LG2(3);
+      if (((1 << field->descriptortype) & fixed64_ok) == 0) goto unknown;
+      break;
+    case UPB_WIRE_TYPE_DELIMITED: {
+      uint32_t size;
+      int ndx = field->descriptortype;
+      if (_upb_isrepeated(field)) ndx += 18;
+      ptr = decode_varint32(d, ptr, d->limit, &size);
+      if (size >= INT32_MAX || (size_t)(d->limit - ptr) < size) {
+        decode_err(d); /* Length overflow. */
       }
-      case UPB_WIRE_TYPE_START_GROUP:
-        val.uint32_val = field_number;
-        op = OP_SUBMSG;
-        if (field->descriptortype != UPB_DTYPE_GROUP) goto unknown;
+      val.str_val.data = ptr;
+      val.str_val.size = size;
+      ptr += size;
+      op = delim_ops[ndx];
+      break;
+    }
+    case UPB_WIRE_TYPE_START_GROUP:
+      val.uint32_val = field_number;
+      op = OP_SUBMSG;
+      if (field->descriptortype != UPB_DTYPE_GROUP) goto unknown;
+      break;
+    case UPB_WIRE_TYPE_END_GROUP:
+      d->end_group = field_number;
+      return ptr;
+    default:
+      decode_err(d);
+  }
+
+  if (op >= 0) {
+    /* Parse, using op for dispatch. */
+    switch (field->label) {
+      case UPB_LABEL_REPEATED:
+      case _UPB_LABEL_PACKED:
+        ptr = decode_toarray(d, ptr, msg, layout, field, val, op);
+        break;
+      case _UPB_LABEL_MAP:
+        decode_tomap(d, msg, layout, field, val);
         break;
-      case UPB_WIRE_TYPE_END_GROUP:
-        d->end_group = field_number;
-        return ptr;
       default:
-        decode_err(d);
+        ptr = decode_tomsg(d, ptr, msg, layout, field, val, op);
+        break;
     }
-
-    if (op >= 0) {
-      /* Parse, using op for dispatch. */
-      switch (field->label) {
-        case UPB_LABEL_REPEATED:
-        case _UPB_LABEL_PACKED:
-          ptr = decode_toarray(d, ptr, msg, layout, field, val, op);
-          break;
-        case _UPB_LABEL_MAP:
-          decode_tomap(d, msg, layout, field, val);
-          break;
-        default:
-          ptr = decode_tomsg(d, ptr, msg, layout, field, val, op);
-          break;
-      }
-    } else {
-    unknown:
-      /* Skip unknown field. */
-      if (field_number == 0) decode_err(d);
-      if (wire_type == UPB_WIRE_TYPE_START_GROUP) {
-        ptr = decode_group(d, ptr, NULL, NULL, field_number);
-      }
-      if (msg) {
-        if (!_upb_msg_addunknown(msg, field_start, ptr - field_start,
-                                 d->arena)) {
-          decode_err(d);
-        }
+  } else {
+  unknown:
+    /* Skip unknown field. */
+    if (field_number == 0) decode_err(d);
+    if (wire_type == UPB_WIRE_TYPE_START_GROUP) {
+      ptr = decode_group(d, ptr, NULL, NULL, field_number);
+    }
+    if (msg) {
+      if (!_upb_msg_addunknown(msg, field_start, ptr - field_start,
+                               d->arena)) {
+        decode_err(d);
       }
     }
   }
 
+  return ptr;
+}
+
+static const char *decode_msg(upb_decstate *d, const char *ptr, upb_msg *msg,
+                              const upb_msglayout *layout) {
+  while (ptr < d->limit) {
+    ptr = decode_field(d, ptr, msg, layout);
+  }
+
   if (ptr != d->limit) decode_err(d);
   return ptr;
 }

upb/decode.h

@@ -16,27 +16,33 @@ extern "C" {
 bool upb_decode(const char *buf, size_t size, upb_msg *msg,
                 const upb_msglayout *l, upb_arena *arena);
 
-/* Internal only: data pertaining to the parse. */
-typedef struct {
-  const char *limit;       /* End of delimited region or end of buffer. */
-  const char *fastlimit;   /* End of delimited region or end of buffer. */
-  upb_arena *arena;
-  int depth;
-  uint32_t end_group; /* Set to field number of END_GROUP tag, if any. */
-  jmp_buf err;
-} upb_decstate;
-
 struct upb_fasttable;
+struct upb_decstate;
 
-typedef const char *_upb_field_parser(upb_decstate *d, const char *ptr,
+typedef const char *_upb_field_parser(struct upb_decstate *d, const char *ptr,
                                       upb_msg *msg, struct upb_fasttable *table,
                                       uint64_t hasbits, uint64_t data);
 
 typedef struct upb_fasttable {
   _upb_field_parser *field_parser[16];
   uint64_t field_data[16];
+  _upb_field_parser *fallback;
 } upb_fasttable;
 
+/* Internal only: data pertaining to the parse. */
+typedef struct upb_decstate {
+  char *arena_ptr, *arena_end;
+  const void *rep_end;
+  const char *limit;       /* End of delimited region or end of buffer. */
+  const char *fastlimit;   /* End of delimited region or end of buffer. */
+  upb_array *arr;
+  _upb_field_parser *resume;
+  upb_arena *arena;
+  int depth;
+  uint32_t end_group; /* Set to field number of END_GROUP tag, if any. */
+  jmp_buf err;
+} upb_decstate;
+
 #ifdef __cplusplus
 }  /* extern "C" */
 #endif

upb/decode_fast.c

@@ -7,54 +7,411 @@
   upb_decstate *d, const char *ptr, upb_msg *msg, upb_fasttable *table, \
       uint64_t hasbits, uint64_t data
 
+#define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data
+
+const char *fastdecode_err(upb_decstate *d);
+
+const char *fastdecode_reallocarr(upb_decstate *d, const char *ptr,
+                                  upb_msg *msg, upb_fasttable *table,
+                                  int elem_size);
+
 UPB_NOINLINE
-const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, upb_msg *msg,
-                                upb_fasttable *table, uint64_t hasbits) {
+static const char *fastdecode_dispatch(upb_decstate *d, const char *ptr,
+                                       upb_msg *msg, upb_fasttable *table,
+                                       uint64_t hasbits) {
   uint16_t tag;
   uint64_t data;
   if (UPB_UNLIKELY(ptr >= d->fastlimit)) return ptr;
   memcpy(&tag, ptr, 2);
   data = table->field_data[(tag & 0xf7) >> 3] ^ tag;
-  return table->field_parser[(tag & 0xf7) >> 3](d, ptr, msg, table, hasbits,
-                                                data);
+  return table->field_parser[(tag & 0xf7) >> 3](UPB_PARSE_ARGS);
+}
+
+#if 0
+UPB_NOINLINE
+static const char *fastdecode_parseloop(upb_decstate *d, const char *ptr,
+                                        upb_msg *msg, upb_fasttable *table) {
+  uint64_t hasbits = 0;
+  while (ptr < d->fastlimit) {
+    ptr = fastdecode_dispatch(d, ptr, msg, table, hasbits);
+    /*ptr = decode_field(d, ptr, msg, table->layout);*/
+  }
+  return ptr;
+}
+#endif
+
+UPB_FORCEINLINE static bool fastdecode_checktag(uint64_t data, int tagbytes) {
+  if (tagbytes == 1) {
+    return (data & 0xff) == 0;
+  } else {
+    return (data & 0xffff) == 0;
+  }
+}
+
+UPB_FORCEINLINE static uint16_t fastdecode_readtag(const char *ptr, int tagbytes) {
+  uint16_t ret = 0;
+  memcpy(&ret, ptr, tagbytes);
+  return ret;
 }
 
-UPB_FORCEINLINE bool fastdecode_checktag(uint64_t data, int tagbytes) {
-  const char zeros[2] = {0, 0};
-  return memcmp(&data, &zeros, tagbytes) == 0;
+typedef enum {
+  CARD_s = 0,
+  CARD_o = 1,
+  CARD_r = 2,
+  CARD_p = 3
+} upb_card;
+
+UPB_FORCEINLINE
+static void *fastdecode_getfield(upb_decstate *d, const char *ptr, upb_msg *msg,
+                                 uint64_t data, uint64_t *hasbits,
+                                 uint16_t *expected_tag, int *elem_avail,
+                                 upb_card card, int tagbytes, int valbytes) {
+  void *field = (char *)msg + (data >> 48);
+
+  switch (card) {
+    case CARD_s:
+      *hasbits |= data;
+      return field;
+    case CARD_o: {
+      uint32_t *case_ptr = UPB_PTR_AT(msg, (data >> 16) & 0xffff, uint32_t);
+      *case_ptr = (data >> 32) & 0xffff;
+      return field;
+    }
+    case CARD_r: {
+      upb_array **arr_p = field;
+      upb_array *arr;
+      *hasbits >>= 16;
+      *(uint32_t*)msg |= *hasbits;
+      *hasbits = 0;
+      if (UPB_LIKELY(!*arr_p)) {
+        //(void)d;
+        size_t need = (valbytes * 4) + sizeof(upb_array);
+        if (UPB_UNLIKELY((size_t)(d->arena_end - d->arena_ptr) < need)) {
+          *elem_avail = 0;
+          return NULL;
+        }
+        arr = (void*)d->arena_ptr;
+        field = arr + 1;
+        arr->data = (uintptr_t)field;
+        *arr_p = arr;
+        arr->size = 4;
+        arr->len = 0;
+        *elem_avail = 4;
+        d->arena_ptr += need;
+      } else {
+        arr = *arr_p;
+        field = _upb_array_ptr(arr);
+        *elem_avail = arr->size - arr->len;
+        field = (char*)field + (arr->len * valbytes);
+        arr->len = arr->size;
+      }
+      *expected_tag = fastdecode_readtag(ptr, tagbytes);
+      d->arr = arr;
+      return field;
+    }
+    default:
+      UPB_UNREACHABLE();
+  }
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_string(UPB_PARSE_PARAMS, int tagbytes,
+                                     upb_card card) {
+  upb_strview *dst;
+  uint16_t expected_tag;
+  int elem_avail;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    return table->fallback(UPB_PARSE_ARGS);
+  }
+
+  dst = fastdecode_getfield(d, ptr, msg, data, &hasbits, &expected_tag,
+                            &elem_avail, card, tagbytes, sizeof(upb_strview));
+
+again:
+  if (card == CARD_r) {
+    if (UPB_UNLIKELY(elem_avail == 0)) {
+      return fastdecode_reallocarr(d, ptr, msg, table, sizeof(upb_strview));
+    }
+  }
+
+  {
+    int64_t len = ptr[tagbytes];
+    if (UPB_UNLIKELY(len < 0)) {
+      if (card == CARD_r) {
+        d->arr->len -= elem_avail;
+      }
+      return ptr;
+    }
+    ptr += tagbytes + 1;
+    dst->data = ptr;
+    dst->size = len;
+    ptr += len;
+    if (UPB_UNLIKELY(ptr > d->limit)) {
+      return fastdecode_err(d);
+    }
+  }
+
+
+  if (card == CARD_r) {
+    if (UPB_LIKELY(ptr < d->fastlimit) &&
+        fastdecode_readtag(ptr, tagbytes) == expected_tag) {
+      elem_avail--;
+      dst++;
+      goto again;
+    }
+    d->arr->len -= elem_avail;
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+const char *upb_pss_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_string(UPB_PARSE_ARGS, 1, CARD_s);
+}
+
+const char *upb_pos_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_string(UPB_PARSE_ARGS, 1, CARD_o);
+}
+
+const char *upb_prs_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_string(UPB_PARSE_ARGS, 1, CARD_r);
+}
+
+UPB_FORCEINLINE
+static const char *fastdecode_fixed(UPB_PARSE_PARAMS, int tagbytes, int valbytes,
+                                    upb_card card) {
+  char *dst;
+  uint16_t expected_tag;
+  int elem_avail;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    return ptr;
+  }
+
+  dst = fastdecode_getfield(d, ptr, msg, data, &hasbits, &expected_tag,
+                            &elem_avail, card, tagbytes, valbytes);
+
+again:
+  if (card == CARD_r) {
+    if (UPB_UNLIKELY(elem_avail == 0)) {
+      return fastdecode_reallocarr(d, ptr, msg, table, valbytes);
+    }
+  }
+
+  {
+    ptr += tagbytes;
+    memcpy(dst, ptr, valbytes);
+    ptr += valbytes;
+  }
+
+  if (card == CARD_r) {
+    if (UPB_LIKELY(ptr < d->fastlimit) &&
+        fastdecode_readtag(ptr, tagbytes) == expected_tag) {
+      elem_avail--;
+      dst += valbytes;
+      goto again;
+    }
+    d->arr->len -= elem_avail;
+  }
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
+}
+
+const char *upb_psf8_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_fixed(UPB_PARSE_ARGS, 1, 8, CARD_s);
+}
+
+const char *upb_pof8_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_fixed(UPB_PARSE_ARGS, 1, 8, CARD_o);
+}
+
+const char *upb_prf8_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_fixed(UPB_PARSE_ARGS, 1, 8, CARD_r);
+}
+
+#if 0
+UPB_FORCEINLINE
+static const char *fastdecode_repeatedfixed(UPB_PARSE_PARAMS, int tagbytes,
+                                            int valbytes, _upb_field_parser *fallback) {
+  char *dst;
+  uint16_t expected_tag;
+  upb_array **arr_p;
+  upb_array *arr;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (fallback) {
+      // Patch data to amtch packed wiretype.
+      data ^= 0x2 ^ (valbytes == 4 ? 5 : 1);
+      fallback(UPB_PARSE_ARGS);
+    } else {
+      return table->fallback(UPB_PARSE_ARGS);
+    }
+  }
+
+  arr_p = UPB_PTR_AT(msg, (data >> 48), upb_array*);
+  arr = *arr_p;
+  if (UPB_UNLIKELY(!arr || arr->size - arr->len < 4)) {
+    return fastdecode_allocarr(UPB_PARSE_ARGS);
+  }
+  dst = _upb_array_ptr(arr);
+  d->dstend = dst + (arr->size * valbytes);
+  dst += (arr->len * valbytes);
+  expected_tag = fastdecode_readtag(ptr, tagbytes);
+
+  do {
+    ptr += tagbytes;
+    //fastdecode_reserve(d, arr, &dst, &dstend);
+    if (UPB_UNLIKELY(dst == d->dstend)) {
+      return fastdecode_reallocarr(UPB_PARSE_ARGS);
+    }
+    memcpy(dst, ptr, valbytes);
+    dst += valbytes;
+    ptr += valbytes;
+    /*
+    if (UPB_UNLIKELY(ptr >= d->fastlimit)) {
+      arr->len = (dst - (char*)_upb_array_ptr(arr)) / valbytes;
+      return ptr;
+    }
+    */
+  } while (fastdecode_readtag(ptr, tagbytes) == expected_tag);
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
 }
 
 UPB_FORCEINLINE
 static const char *fastdecode_scalarfixed(UPB_PARSE_PARAMS, int tagbytes,
-                                    int valbytes) {
+                                          int valbytes, upb_card card) {
   char *field;
-  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) return ptr;
-  hasbits |= data;
-  field = (char*)msg + (data >> 48);
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    return table->fallback(UPB_PARSE_ARGS);
+  }
+  field = fastdecode_getfield(msg, data, &hasbits, card);
   memcpy(field, ptr + tagbytes, valbytes);
-  return fastdecode_dispatch(d, ptr + tagbytes + valbytes, msg, table, hasbits);
+  ptr += tagbytes + valbytes;
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
 }
 
-const char *upb_psf64_1bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarfixed(d, ptr, msg, table, hasbits, data, 1, 8);
+  arr_p = UPB_PTR_AT(msg, (data >> 48), upb_array*);
+  if (UPB_UNLIKELY(!arr_p)) goto alloc_arr;
+  arr = *arr_p;
+  dst = (char*)_upb_array_ptr(arr);
+  dstend = dst + arr->size;
+  dst += arr->len;
+
+const char *fastdecode_allocarr(UPB_PARSE_PARAMS)
+  ;
+
+UPB_FORCEINLINE
+static void fastdecode_getarr(upb_decstate *d, upb_msg *msg, uint64_t data,
+                              int valbytes, char **dst) {
+  upb_array **arr_p = UPB_PTR_AT(msg, (data >> 48), upb_array*);
+  upb_array *arr = *arr_p;
+  /*
+  if (UPB_UNLIKELY(!arr || arr->size - arr->len < 4)) {
+    fastdecode_allocarr(d, arr_p);
+  }
+  */
+  (void)d;
+  *dst = _upb_array_ptr(arr);
+  d->dstend = *dst + (arr->size * valbytes);
+  *dst += (arr->len * valbytes);
 }
 
-const char *upb_psf64_2bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarfixed(d, ptr, msg, table, hasbits, data, 2, 8);
+UPB_FORCEINLINE
+static const char *fastdecode_repeatedfixed(UPB_PARSE_PARAMS, int tagbytes,
+                                            int valbytes, _upb_field_parser *fallback) {
+  char *dst;
+  uint16_t expected_tag;
+  upb_array **arr_p;
+  upb_array *arr;
+
+  if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) {
+    if (fallback) {
+      // Patch data to amtch packed wiretype.
+      data ^= 0x2 ^ (valbytes == 4 ? 5 : 1);
+      fallback(UPB_PARSE_ARGS);
+    } else {
+      return table->fallback(UPB_PARSE_ARGS);
+    }
+  }
+
+  arr_p = UPB_PTR_AT(msg, (data >> 48), upb_array*);
+  arr = *arr_p;
+  if (UPB_UNLIKELY(!arr || arr->size - arr->len < 4)) {
+    return fastdecode_allocarr(UPB_PARSE_ARGS);
+  }
+  dst = _upb_array_ptr(arr);
+  d->dstend = dst + (arr->size * valbytes);
+  dst += (arr->len * valbytes);
+  expected_tag = fastdecode_readtag(ptr, tagbytes);
+
+  do {
+    ptr += tagbytes;
+    //fastdecode_reserve(d, arr, &dst, &dstend);
+    if (UPB_UNLIKELY(dst == d->dstend)) {
+      return fastdecode_reallocarr(UPB_PARSE_ARGS);
+    }
+    memcpy(dst, ptr, valbytes);
+    dst += valbytes;
+    ptr += valbytes;
+    /*
+    if (UPB_UNLIKELY(ptr >= d->fastlimit)) {
+      arr->len = (dst - (char*)_upb_array_ptr(arr)) / valbytes;
+      return ptr;
+    }
+    */
+  } while (fastdecode_readtag(ptr, tagbytes) == expected_tag);
+
+  return fastdecode_dispatch(d, ptr, msg, table, hasbits);
 }
 
-const char *upb_psf32_1bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarfixed(d, ptr, msg, table, hasbits, data, 1, 4);
+UPB_NOINLINE
+const char *upb_prf8_1bt(UPB_PARSE_PARAMS) {
+  return fastdecode_repeatedfixed(UPB_PARSE_ARGS, 1, 8, false);
 }
 
-const char *upb_psf32_2bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarfixed(d, ptr, msg, table, hasbits, data, 2, 4);
+
+// Generate all fixed functions.
+// {s,o,r,p} x {f4,f8} x {1bt,2bt}
+
+#define F(card, valbytes, tagbytes)                                           \
+  const char *upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) {   \
+    return fastdecode_fixed(UPB_PARSE_ARGS, tagbytes, valbytes, CARD_##card); \
+  }
+
+#define TYPES(card, tagbytes) \
+  F(card, 4, tagbytes)        \
+  F(card, 8, tagbytes)
+
+#define TAGBYTES(card) \
+  TYPES(card, 1)       \
+  TYPES(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+TAGBYTES(p)
+
+
+UPB_FORCEINLINE uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) {
+  if (valbytes == 1) {
+    return val != 0;
+  } else if (zigzag) {
+    if (valbytes == 4) {
+      uint32_t n = val;
+      return (n >> 1) ^ -(int32_t)(n & 1);
+    } else if (valbytes == 8) {
+      return (val >> 1) ^ -(int64_t)(val & 1);
+    }
+    UPB_UNREACHABLE();
+  }
+  return val;
 }
 
 UPB_FORCEINLINE
-static const char *fastdecode_longvarint_impl(UPB_PARSE_PARAMS, int valbytes) {
-  char *field = (char *)data;
-
+static const char *fastdecode_longvarint_impl(const char *ptr, void *field,
+                                              int valbytes) {
   // The algorithm relies on sign extension to set all high bits when the varint
   // continues. This way it can use "and" to aggregate in to the result.
   const int8_t *p = (const int8_t*)(ptr);
@@ -131,17 +488,17 @@ static const char *fastdecode_longvarint_impl(UPB_PARSE_PARAMS, int valbytes) {
 
   return NULL;  // Value is too long to be a varint64
 
-#define DONE(n)                                                              \
-  done##n : {                                                                \
-    uint64_t val = res1 & res2 & res3;                                       \
-    memcpy(field, &val, valbytes);                                           \
-    return fastdecode_dispatch(d, (const char *)p + n, msg, table, hasbits); \
+#define DONE(n)                        \
+  done##n : {                          \
+    uint64_t val = res1 & res2 & res3; \
+    memcpy(field, &val, valbytes);     \
+    return (const char *)p + n;        \
   };
 
 done2 : {
   uint64_t val = res1 & res2;
   memcpy(field, &val, valbytes);
-  return fastdecode_dispatch(d, (const char*)p + 2, msg, table, hasbits);
+  return (const char*)p + 2;
 }
 
   DONE(3)
@@ -152,59 +509,94 @@ done2 : {
   DONE(8)
   DONE(9)
   DONE(10)
+
 #undef DONE
+#undef SHLD
+#undef SHLD_SIGN
 }
 
 UPB_NOINLINE
 static const char *fastdecode_longvarint32(UPB_PARSE_PARAMS) {
-  return fastdecode_longvarint_impl(d, ptr, msg, table, hasbits, data, 4);
+  (void)d;
+  (void)msg;
+  (void)table;
+  (void)hasbits;
+  return fastdecode_longvarint_impl(ptr, (void*)data, 4);
 }
 
 UPB_NOINLINE
 static const char *fastdecode_longvarint64(UPB_PARSE_PARAMS) {
-  return fastdecode_longvarint_impl(d, ptr, msg, table, hasbits, data, 8);
+  (void)d;
+  (void)msg;
+  (void)table;
+  (void)hasbits;
+  return fastdecode_longvarint_impl(ptr, (void*)data, 8);
 }
 
 UPB_FORCEINLINE
 static const char *fastdecode_longvarint(UPB_PARSE_PARAMS, int valbytes) {
   if (valbytes == 4) {
-    return fastdecode_longvarint32(d, ptr, msg, table, hasbits, data);
+    return fastdecode_longvarint32(UPB_PARSE_ARGS);
   } else if (valbytes == 8) {
-    return fastdecode_longvarint64(d, ptr, msg, table, hasbits, data);
+    return fastdecode_longvarint64(UPB_PARSE_ARGS);
   }
   UPB_UNREACHABLE();
 }
 
 UPB_FORCEINLINE
-static const char *fastdecode_scalarvarint(UPB_PARSE_PARAMS, int tagbytes,
-                                           int valbytes) {
+static const char *fastdecode_varint(UPB_PARSE_PARAMS, int tagbytes,
+                                     int valbytes, bool zigzag, bool oneof) {
   uint64_t val = 0;
   void *field;
   if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) return ptr;
   ptr += tagbytes;
-  hasbits |= data;
+  fastdecode_getfield(msg, data, &hasbits, oneof);
   field = (char*)msg + (data >> 48);
   if (UPB_UNLIKELY(*ptr < 0)) {
     return fastdecode_longvarint(d, ptr, msg, table, hasbits, (uint64_t)field,
                                  valbytes);
   }
-  val = *ptr;
+  val = fastdecode_munge(*ptr, valbytes, zigzag);
   memcpy(field, &val, valbytes);
   return fastdecode_dispatch(d, ptr + 1, msg, table, hasbits);
 }
 
-const char *upb_psv32_1bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarvarint(d, ptr, msg, table, hasbits, data, 1, 4);
-}
+// Generate all varint functions.
+// {s,o,r} x {b1,v4,z4,v8,z8} x {1bt,2bt}
 
-const char *upb_psv32_2bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarvarint(d, ptr, msg, table, hasbits, data, 2, 4);
-}
+#define z_ZZ true
+#define b_ZZ false
+#define v_ZZ false
 
-const char *upb_psv64_1bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarvarint(d, ptr, msg, table, hasbits, data, 1, 8);
-}
+#define F(card, type, valbytes, tagbytes)                                      \
+  const char *upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
+    return fastdecode_varint(UPB_PARSE_ARGS, tagbytes, valbytes, type##_ZZ,    \
+                             card##_ONEOF);                                    \
+  }
 
-const char *upb_psv64_2bt(UPB_PARSE_PARAMS) {
-  return fastdecode_scalarvarint(d, ptr, msg, table, hasbits, data, 2, 8);
-}
+#define TYPES(card, tagbytes) \
+  F(card, b, 1, tagbytes)     \
+  F(card, v, 4, tagbytes)     \
+  F(card, v, 8, tagbytes)     \
+  F(card, z, 4, tagbytes)     \
+  F(card, z, 8, tagbytes)
+
+#define TAGBYTES(card) \
+  TYPES(card, 1)       \
+  TYPES(card, 2)
+
+TAGBYTES(s)
+TAGBYTES(o)
+TAGBYTES(r)
+
+#undef z_ZZ
+#undef b_ZZ
+#undef v_ZZ
+#undef o_ONEOF
+#undef s_ONEOF
+#undef r_ONEOF
+#undef F
+#undef TYPES
+#undef TAGBYTES
+
+#endif