upb_def compiles again!

core/upb.c

@@ -45,7 +45,7 @@ void upb_seterr(upb_status *status, enum upb_status_code code,
 {
   if(upb_ok(status)) {  // The first error is the most interesting.
     status->code = code;
-    status->str = upb_string_tryrecycle(status->str);
+    upb_string_recycle(&status->str);
     va_list args;
     va_start(args, msg);
     upb_string_vprintf(status->str, msg, args);

core/upb.h

@@ -126,14 +126,20 @@ struct _upb_array;
 typedef struct _upb_array upb_array;
 struct _upb_msg;
 typedef struct _upb_msg upb_msg;
+struct _upb_bytesrc;
+typedef struct _upb_bytesrc upb_bytesrc;
 
-typedef uint32_t upb_strlen_t;
+typedef int32_t upb_strlen_t;
+#define UPB_STRLEN_MAX INT32_MAX
 
 // The type of a upb_value.  This is like a upb_fieldtype_t, but adds the
 // constant UPB_VALUETYPE_ARRAY to represent an array.
 typedef uint8_t upb_valuetype_t;
 #define UPB_VALUETYPE_ARRAY 32
 
+#define UPB_VALUETYPE_BYTESRC 32
+#define UPB_VALUETYPE_RAW 33
+
 // A single .proto value.  The owner must have an out-of-band way of knowing
 // the type, so that it knows which union member to use.
 typedef struct {
@@ -146,6 +152,7 @@ typedef struct {
     uint64_t uint64;
     bool _bool;
     upb_string *str;
+    upb_bytesrc *bytesrc;
     upb_msg *msg;
     upb_array *arr;
     upb_atomic_refcount_t *refcount;
@@ -167,21 +174,27 @@ typedef struct {
 
 #define UPB_VALUE_ACCESSORS(name, membername, ctype, proto_type) \
   ctype upb_value_get ## name(upb_value val) { \
-    assert(val.type == UPB_TYPE(proto_type)); \
+    assert(val.type == proto_type || val.type == UPB_VALUETYPE_RAW); \
     return val.val.membername; \
   } \
-  void upb_value_ ## name(upb_value *val, ctype cval) { \
-    SET_TYPE(val->type, UPB_TYPE(proto_type)); \
+  void upb_value_set ## name(upb_value *val, ctype cval) { \
+    SET_TYPE(val->type, proto_type); \
     val->val.membername = cval; \
   }
-UPB_VALUE_ACCESSORS(double, _double, double, DOUBLE);
-UPB_VALUE_ACCESSORS(float, _float, float, FLOAT);
-UPB_VALUE_ACCESSORS(int32, int32, int32_t, INT32);
-UPB_VALUE_ACCESSORS(int64, int64, int64_t, INT64);
-UPB_VALUE_ACCESSORS(uint32, uint32, uint32_t, UINT32);
-UPB_VALUE_ACCESSORS(uint64, uint64, uint64_t, UINT64);
-UPB_VALUE_ACCESSORS(bool, _bool, bool, BOOL);
-UPB_VALUE_ACCESSORS(str, str, upb_string*, STRING);
+UPB_VALUE_ACCESSORS(double, _double, double, UPB_TYPE(DOUBLE));
+UPB_VALUE_ACCESSORS(float, _float, float, UPB_TYPE(FLOAT));
+UPB_VALUE_ACCESSORS(int32, int32, int32_t, UPB_TYPE(INT32));
+UPB_VALUE_ACCESSORS(int64, int64, int64_t, UPB_TYPE(INT64));
+UPB_VALUE_ACCESSORS(uint32, uint32, uint32_t, UPB_TYPE(UINT32));
+UPB_VALUE_ACCESSORS(uint64, uint64, uint64_t, UPB_TYPE(UINT64));
+UPB_VALUE_ACCESSORS(bool, _bool, bool, UPB_TYPE(BOOL));
+UPB_VALUE_ACCESSORS(str, str, upb_string*, UPB_TYPE(STRING));
+UPB_VALUE_ACCESSORS(bytesrc, bytesrc, upb_bytesrc*, UPB_VALUETYPE_BYTESRC);
+
+void upb_value_setraw(upb_value *val, uint64_t cval) {
+  SET_TYPE(val->type, UPB_VALUETYPE_RAW);
+  val->val.uint64 = cval;
+}
 
 // A pointer to a .proto value.  The owner must have an out-of-band way of
 // knowing the type, so it knows which union member to use.

core/upb_def.c

@@ -228,6 +228,10 @@ static void upb_deflist_push(upb_deflist *l, upb_def *d) {
   l->defs[l->len++] = d;
 }
 
+static upb_def *upb_deflist_last(upb_deflist *l) {
+  return l->defs[l->len-1];
+}
+
 // Qualify the defname for all defs starting with offset "start" with "str".
 static void upb_deflist_qualify(upb_deflist *l, upb_string *str, int32_t start) {
   for(uint32_t i = start; i < l->len; i++) {
@@ -238,8 +242,14 @@ static void upb_deflist_qualify(upb_deflist *l, upb_string *str, int32_t start)
   }
 }
 
+// We keep a stack of all the messages scopes we are currently in, as well as
+// the top-level file scope.  This is necessary to correctly qualify the
+// definitions that are contained inside.  "name" tracks the name of the
+// message or package (a bare name -- not qualified by any enclosing scopes).
 typedef struct {
   upb_string *name;
+  // Index of the first def that is under this scope.  For msgdefs, the
+  // msgdef itself is at start-1.
   int start;
 } upb_defbuilder_frame;
 
@@ -250,6 +260,10 @@ struct _upb_defbuilder {
 
   uint32_t number;
   upb_string *name;
+  bool saw_number;
+  bool saw_name;
+
+  upb_fielddef *f;
 };
 typedef struct _upb_defbuilder upb_defbuilder;
 
@@ -259,6 +273,28 @@ static void upb_enumdef_register_EnumDescriptorProto(upb_defbuilder *b,
                                                      upb_handlers *h);
 
 
+static void upb_defbuilder_init(upb_defbuilder *b) {
+  upb_deflist_init(&b->defs);
+  b->stack_len = 0;
+  b->name = NULL;
+}
+
+static void upb_defbuilder_uninit(upb_defbuilder *b) {
+  upb_string_unref(b->name);
+  upb_deflist_uninit(&b->defs);
+}
+
+static upb_msgdef *upb_defbuilder_top(upb_defbuilder *b) {
+  if (b->stack_len <= 1) return NULL;
+  int index = b->stack[b->stack_len-1].start - 1;
+  assert(index >= 0);
+  return upb_downcast_msgdef(b->defs.defs[index]);
+}
+
+static upb_def *upb_defbuilder_last(upb_defbuilder *b) {
+  return upb_deflist_last(&b->defs);
+}
+
 // Start/end handlers for FileDescriptorProto and DescriptorProto (the two
 // entities that have names and can contain sub-definitions.
 void upb_defbuilder_startcontainer(upb_defbuilder *b) {
@@ -291,9 +327,8 @@ static upb_flow_t upb_defbuilder_FileDescriptorProto_value(void *_b,
     case GOOGLE_PROTOBUF_FILEDESCRIPTORPROTO_MESSAGE_TYPE_FIELDNUM:
     case GOOGLE_PROTOBUF_FILEDESCRIPTORPROTO_ENUM_TYPE_FIELDNUM:
       return BEGIN_SUBMSG;
-    default:
-      return UPB_SKIP;
   }
+  return UPB_CONTINUE;
 }
 
 static upb_flow_t upb_defbuilder_FileDescriptorProto_startsubmsg(
@@ -308,19 +343,19 @@ static upb_flow_t upb_defbuilder_FileDescriptorProto_startsubmsg(
       return UPB_DELEGATE;
     default:
       // TODO: services and extensions.
-      return UPB_SKIP;
+      return UPB_SKIPSUBMSG;
   }
 }
 
 static void upb_defbuilder_register_FileDescriptorProto(upb_defbuilder *b,
                                                         upb_handlers *h) {
-  static upb_handlerset upb_defbuilder_FileDescriptorProto_handlers = {
+  static upb_handlerset handlers = {
     NULL,  // startmsg
     NULL,  // endmsg
     &upb_defbuilder_FileDescriptorProto_value,
     &upb_defbuilder_FileDescriptorProto_startsubmsg,
   };
-  upb_register_handlerset(h, &upb_defbuilder_FileDescriptorProto_handlers);
+  upb_register_handlerset(h, &handlers);
   upb_set_handler_closure(h, b);
 }
 
@@ -333,9 +368,8 @@ static upb_flow_t upb_defbuilder_FileDescriptorSet_value(void *b,
   switch(f->number) {
     case GOOGLE_PROTOBUF_FILEDESCRIPTORSET_FILE_FIELDNUM:
       return BEGIN_SUBMSG;
-    default:
-      return UPB_SKIP;
   }
+  return UPB_CONTINUE;
 }
 
 static upb_flow_t upb_defbuilder_FileDescriptorSet_startsubmsg(
@@ -345,20 +379,19 @@ static upb_flow_t upb_defbuilder_FileDescriptorSet_startsubmsg(
     case GOOGLE_PROTOBUF_FILEDESCRIPTORSET_FILE_FIELDNUM:
       upb_defbuilder_register_FileDescriptorProto(b, h);
       return UPB_DELEGATE;
-    default:
-      return UPB_SKIP;
   }
+  return UPB_SKIPSUBMSG;
 }
 
 static void upb_defbuilder_register_FileDescriptorSet(
     upb_defbuilder *b, upb_handlers *h) {
-  static upb_handlerset upb_defbuilder_FileDescriptorSet_handlers = {
+  static upb_handlerset handlers = {
     NULL,  // startmsg
     NULL,  // endmsg
     &upb_defbuilder_FileDescriptorSet_value,
     &upb_defbuilder_FileDescriptorSet_startsubmsg,
   };
-  upb_register_handlerset(h, &upb_defbuilder_FileDescriptorSet_handlers);
+  upb_register_handlerset(h, &handlers);
   upb_set_handler_closure(h, b);
 }
 
@@ -406,18 +439,20 @@ static void upb_enumdef_free(upb_enumdef *e) {
 }
 
 // google.protobuf.EnumValueDescriptorProto.
-static void upb_enumdef_EnumValueDescriptorProto_startmsg(upb_defbuilder *b) {
-  b->number = -1;
-  b->name = NULL;
+static void upb_enumdef_EnumValueDescriptorProto_startmsg(void *_b) {
+  upb_defbuilder *b = _b;
+  b->saw_number = false;
+  b->saw_name = false;
 }
 
-static upb_flow_t upb_enumdef_EnumValueDescriptorProto_value(upb_defbuilder *b,
+static upb_flow_t upb_enumdef_EnumValueDescriptorProto_value(void *_b,
                                                              upb_fielddef *f,
                                                              upb_value val) {
+  upb_defbuilder *b = _b;
   switch(f->number) {
     case GOOGLE_PROTOBUF_ENUMVALUEDESCRIPTORPROTO_NAME_FIELDNUM:
-      b->name = upb_string_tryrecycle(name);
-      CHECKSRC(upb_src_getstr(src, name));
+      upb_string_unref(b->name);
+      upb_string_getref(upb_value_getstr(val));
       break;
     case GOOGLE_PROTOBUF_ENUMVALUEDESCRIPTORPROTO_NUMBER_FIELDNUM:
       b->number = upb_value_getint32(val);
@@ -428,34 +463,37 @@ static upb_flow_t upb_enumdef_EnumValueDescriptorProto_value(upb_defbuilder *b,
   return UPB_CONTINUE;
 }
 
-static void upb_enumdef_EnumValueDescriptorProto_endmsg(upb_defbuilder *b) {
-  if(b->name == NULL || b->number == -1) {
-    upb_seterr(status, UPB_STATUS_ERROR, "Enum value missing name or number.");
-    goto err;
+static void upb_enumdef_EnumValueDescriptorProto_endmsg(void *_b) {
+  upb_defbuilder *b = _b;
+  if(!b->saw_number || !b->saw_name) {
+    //upb_seterr(status, UPB_STATUS_ERROR, "Enum value missing name or number.");
+    //goto err;
+    return;
   }
-  upb_ntoi_ent ntoi_ent = {{name, 0}, number};
-  upb_iton_ent iton_ent = {{number, 0}, name};
+  upb_ntoi_ent ntoi_ent = {{b->name, 0}, b->number};
+  upb_iton_ent iton_ent = {{b->number, 0}, b->name};
+  upb_enumdef *e = upb_downcast_enumdef(upb_defbuilder_last(b));
   upb_strtable_insert(&e->ntoi, &ntoi_ent.e);
   upb_inttable_insert(&e->iton, &iton_ent.e);
   // We don't unref "name" because we pass our ref to the iton entry of the
   // table.  strtables can ref their keys, but the inttable doesn't know that
   // the value is a string.
-  return UPB_CONTINUE;
 }
 
 static void upb_enumdef_register_EnumValueDescriptorProto(upb_defbuilder *b,
                                                           upb_handlers *h) {
-  static upb_handlerset upb_enumdef_EnumValueDescriptorProto_handlers = {
+  static upb_handlerset handlers = {
     &upb_enumdef_EnumValueDescriptorProto_startmsg,
     &upb_enumdef_EnumValueDescriptorProto_endmsg,
     &upb_enumdef_EnumValueDescriptorProto_value,
-  }
-  upb_register_handlerset(h, &upb_enumdef_EnumValueDescriptorProto_handlers);
+  };
+  upb_register_handlerset(h, &handlers);
   upb_set_handler_closure(h, b);
 }
 
 // google.protobuf.EnumDescriptorProto.
-void upb_enumdef_EnumDescriptorProto_startmsg(upb_defbuilder *b) {
+void upb_enumdef_EnumDescriptorProto_startmsg(void *_b) {
+  upb_defbuilder *b = _b;
   upb_enumdef *e = malloc(sizeof(*e));
   upb_def_init(&e->base, UPB_DEF_ENUM);
   upb_strtable_init(&e->ntoi, 0, sizeof(upb_ntoi_ent));
@@ -463,42 +501,51 @@ void upb_enumdef_EnumDescriptorProto_startmsg(upb_defbuilder *b) {
   upb_deflist_push(&b->defs, UPB_UPCAST(e));
 }
 
-void upb_enumdef_EnumDescriptorProto_endmsg(upb_defbuilder *b) {
-  assert(e->base.fqname);
+void upb_enumdef_EnumDescriptorProto_endmsg(void *_b) {
+  upb_defbuilder *b = _b;
+  assert(upb_defbuilder_last(b)->fqname != NULL);
 }
 
-static upb_flow_t upb_enumdef_EnumDescriptorProto_value(upb_defbuilder *b,
+static upb_flow_t upb_enumdef_EnumDescriptorProto_value(void *_b,
                                                         upb_fielddef *f,
                                                         upb_value val) {
+  upb_defbuilder *b = _b;
   switch(f->number) {
-    case GOOGLE_PROTOBUF_ENUMDESCRIPTORPROTO_NAME_FIELDNUM:
+    case GOOGLE_PROTOBUF_ENUMDESCRIPTORPROTO_NAME_FIELDNUM: {
+      upb_enumdef *e = upb_downcast_enumdef(upb_defbuilder_last(b));
       upb_string_unref(e->base.fqname);
-      e->base.fqname = upb_value_getstr(val);
+      e->base.fqname = upb_string_getref(upb_value_getstr(val));
+      return UPB_CONTINUE;
+    }
     case GOOGLE_PROTOBUF_ENUMDESCRIPTORPROTO_VALUE_FIELDNUM:
       return BEGIN_SUBMSG;
+    default:
+      return UPB_CONTINUE;
   }
-  return UPB_CONTINUE;
 }
 
-static upb_flow_t upb_enumdef_EnumDescriptorProto_startsubmsg(upb_defbuilder *b,
+static upb_flow_t upb_enumdef_EnumDescriptorProto_startsubmsg(void *_b,
                                                               upb_fielddef *f,
                                                               upb_handlers *h) {
+  upb_defbuilder *b = _b;
   switch(f->number) {
     case GOOGLE_PROTOBUF_ENUMDESCRIPTORPROTO_VALUE_FIELDNUM:
       upb_enumdef_register_EnumValueDescriptorProto(b, h);
       return UPB_DELEGATE;
+    default:
+      return UPB_SKIPSUBMSG;
   }
-  return UPB_SKIP;
 }
 
 static void upb_enumdef_register_EnumDescriptorProto(upb_defbuilder *b,
                                                      upb_handlers *h) {
-  static upb_handlerset upb_enumdef_EnumDescriptorProto_handlers = {
+  static upb_handlerset handlers = {
     &upb_enumdef_EnumDescriptorProto_startmsg,
     &upb_enumdef_EnumDescriptorProto_endmsg,
     &upb_enumdef_EnumDescriptorProto_value,
-  }
-  upb_register_handlerset(h, &upb_enumdef_EnumDescriptorProto_handlers);
+    &upb_enumdef_EnumDescriptorProto_startsubmsg,
+  };
+  upb_register_handlerset(h, &handlers);
   upb_set_handler_closure(h, b);
 }
 
@@ -529,56 +576,71 @@ static void upb_fielddef_free(upb_fielddef *f) {
   free(f);
 }
 
-static void upb_fielddef_startmsg(upb_defbuilder *b) {
+static void upb_fielddef_startmsg(void *_b) {
+  upb_defbuilder *b = _b;
   upb_fielddef *f = malloc(sizeof(*f));
   f->number = -1;
   f->name = NULL;
   f->def = NULL;
   f->owned = false;
-  f->msgdef = m;
+  f->msgdef = upb_defbuilder_top(b);
   b->f = f;
 }
 
-static void upb_fielddef_endmsg(upb_defbuilder *b) {
+static void upb_fielddef_endmsg(void *_b) {
+  upb_defbuilder *b = _b;
+  upb_fielddef *f = b->f;
   // TODO: verify that all required fields were present.
   assert(f->number != -1 && f->name != NULL);
   assert((f->def != NULL) == upb_hasdef(f));
 
   // Field was successfully read, add it as a field of the msgdef.
+  upb_msgdef *m = upb_defbuilder_top(b);
   upb_itof_ent itof_ent = {{f->number, 0}, f};
   upb_ntof_ent ntof_ent = {{f->name, 0}, f};
   upb_inttable_insert(&m->itof, &itof_ent.e);
   upb_strtable_insert(&m->ntof, &ntof_ent.e);
-  return true;
 }
 
-static upb_flow_t upb_fielddef_value(upb_defbuilder *b, upb_fielddef *f, upb_value val) {
-  switch(parsed_f->number) {
+static upb_flow_t upb_fielddef_value(void *_b, upb_fielddef *f, upb_value val) {
+  upb_defbuilder *b = _b;
+  switch(f->number) {
     case GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_FIELDNUM:
-      f->type = upb_value_getint32(val);
+      b->f->type = upb_value_getint32(val);
       break;
     case GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_LABEL_FIELDNUM:
-      f->label = upb_value_getint32(val);
+      b->f->label = upb_value_getint32(val);
       break;
     case GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_NUMBER_FIELDNUM:
-      f->number = upb_value_getint32(val);
+      b->f->number = upb_value_getint32(val);
       break;
     case GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_NAME_FIELDNUM:
-      f->name = upb_string_tryrecycle(f->name);
-      CHECKSRC(upb_src_getstr(src, f->name));
+      upb_string_unref(b->f->name);
+      b->f->name = upb_string_getref(upb_value_getstr(val));
       break;
     case GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_NAME_FIELDNUM: {
       upb_string *str = upb_string_new();
-      CHECKSRC(upb_src_getstr(src, str));
-      if(f->def) upb_def_unref(f->def);
-      f->def = UPB_UPCAST(upb_unresolveddef_new(str));
-      f->owned = true;
+      if (!upb_value_getfullstr(val, str, NULL)) return UPB_ERROR;
+      if(b->f->def) upb_def_unref(b->f->def);
+      b->f->def = UPB_UPCAST(upb_unresolveddef_new(str));
+      b->f->owned = true;
       break;
     }
   }
   return UPB_CONTINUE;
 }
 
+static void upb_fielddef_register_FieldDescriptorProto(upb_defbuilder *b,
+                                                       upb_handlers *h) {
+  static upb_handlerset handlers = {
+    &upb_fielddef_startmsg,
+    &upb_fielddef_endmsg,
+    &upb_fielddef_value,
+  };
+  upb_register_handlerset(h, &handlers);
+  upb_set_handler_closure(h, b);
+}
+
 
 /* upb_msgdef *****************************************************************/
 
@@ -596,21 +658,24 @@ static int upb_compare_fields(const void *f1, const void *f2) {
 }
 
 // google.protobuf.DescriptorProto.
-static void upb_msgdef_startmsg(upb_defbuilder *b) {
+static void upb_msgdef_startmsg(void *_b) {
+  upb_defbuilder *b = _b;
   upb_msgdef *m = malloc(sizeof(*m));
   upb_def_init(&m->base, UPB_DEF_MSG);
   upb_atomic_refcount_init(&m->cycle_refcount, 0);
   upb_inttable_init(&m->itof, 4, sizeof(upb_itof_ent));
   upb_strtable_init(&m->ntof, 4, sizeof(upb_ntof_ent));
   upb_deflist_push(&b->defs, UPB_UPCAST(m));
-  upb_defbuilder_startcontainer(b, UPB_UPCAST(m));
+  upb_defbuilder_startcontainer(b);
 }
 
-static void upb_msgdef_endmsg(upb_defbuilder *b) {
-  upb_msgdef *m = upb_downcast_msgdef(upb_deflist_stacktop(&m->defs));
+static void upb_msgdef_endmsg(void *_b) {
+  upb_defbuilder *b = _b;
+  upb_msgdef *m = upb_defbuilder_top(b);
   if(!m->base.fqname) {
-    upb_seterr(status, UPB_STATUS_ERROR, "Encountered message with no name.");
-    return UPB_ERROR;
+    //upb_seterr(status, UPB_STATUS_ERROR, "Encountered message with no name.");
+    //return UPB_ERROR;
+    return;
   }
 
   // Create an ordering over the fields.
@@ -651,51 +716,57 @@ static void upb_msgdef_endmsg(upb_defbuilder *b) {
   if (max_align > 0) m->size = upb_align_up(m->size, max_align);
 
   upb_defbuilder_endcontainer(b);
-  return UPB_CONTINUE;
+  //return UPB_CONTINUE;
 }
 
-static bool upb_msgdef_value(upb_defbuilder *b, upb_fielddef *f, upb_value val) {
+static upb_flow_t upb_msgdef_value(void *_b, upb_fielddef *f, upb_value val) {
+  upb_defbuilder *b = _b;
   switch(f->number) {
-    case GOOGLE_PROTOBUF_DESCRIPTORPROTO_NAME_FIELDNUM:
-      upb_defbuilder_setscopename(upb_value_getstr(val));
-      break;
+    case GOOGLE_PROTOBUF_DESCRIPTORPROTO_NAME_FIELDNUM: {
+      upb_msgdef *m = upb_defbuilder_top(b);
+      upb_string_unref(m->base.fqname);
+      m->base.fqname = upb_string_getref(upb_value_getstr(val));
+      upb_defbuilder_setscopename(b, upb_value_getstr(val));
+      return UPB_CONTINUE;
+    }
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_FIELD_FIELDNUM:
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_NESTED_TYPE_FIELDNUM:
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_ENUM_TYPE_FIELDNUM:
       return BEGIN_SUBMSG;
     default:
       // TODO: extensions.
-      return UPB_SKIP;
+      return UPB_CONTINUE;
   }
 }
 
-static upb_flow_t upb_msgdef_startsubmsg(upb_defbuilder *b, upb_fielddef *f,
+static upb_flow_t upb_msgdef_startsubmsg(void *_b, upb_fielddef *f,
                                          upb_handlers *h) {
+  upb_defbuilder *b = _b;
   switch(f->number) {
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_FIELD_FIELDNUM:
-      upb_register_FieldDescriptorProto(b, h);
+      upb_fielddef_register_FieldDescriptorProto(b, h);
       return UPB_DELEGATE;
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_NESTED_TYPE_FIELDNUM:
       upb_msgdef_register_DescriptorProto(b, h);
       return UPB_DELEGATE;
     case GOOGLE_PROTOBUF_DESCRIPTORPROTO_ENUM_TYPE_FIELDNUM:
-      upb_register_EnumDescriptorProto(b, h);
+      upb_enumdef_register_EnumDescriptorProto(b, h);
       return UPB_DELEGATE;
       break;
     default:
-      return UPB_SKIP;
+      return UPB_SKIPSUBMSG;
   }
 }
 
 static void upb_msgdef_register_DescriptorProto(upb_defbuilder *b,
                                                 upb_handlers *h) {
-  static upb_handlerset upb_msgdef_DescriptorProto_handlers = {
+  static upb_handlerset handlers = {
     &upb_msgdef_startmsg,
     &upb_msgdef_endmsg,
     &upb_msgdef_value,
     &upb_msgdef_startsubmsg,
-  }
-  upb_register_handlerset(h, &upb_msgdef_DescriptorProto_handlers);
+  };
+  upb_register_handlerset(h, &handlers);
   upb_set_handler_closure(h, b);
 }
 
@@ -884,7 +955,7 @@ bool upb_resolverefs(upb_strtable *tmptab, upb_strtable *symtab,
 // indicating whether the new defs can overwrite existing defs in the symtab,
 // attempts to add the given defs to the symtab.  The whole operation either
 // succeeds or fails.  Ownership of "defs" and "exts" is taken.
-bool upb_symtab_add_defs(upb_symtab *s, upb_defs **defs, int num_defs,
+bool upb_symtab_add_defs(upb_symtab *s, upb_def **defs, int num_defs,
                          bool allow_redef, upb_status *status)
 {
   upb_rwlock_wrlock(&s->lock);
@@ -892,9 +963,9 @@ bool upb_symtab_add_defs(upb_symtab *s, upb_defs **defs, int num_defs,
   // Build a table of the defs we mean to add, for duplicate detection and name
   // resolution.
   upb_strtable tmptab;
-  upb_strtable_init(&tmptab, defs->len, sizeof(upb_symtab_ent));
-  for (uint32_t i = 0; i < defs->len; i++) {
-    upb_def *def = defs->defs[i];
+  upb_strtable_init(&tmptab, num_defs, sizeof(upb_symtab_ent));
+  for (int i = 0; i < num_defs; i++) {
+    upb_def *def = defs[i];
     upb_symtab_ent e = {{def->fqname, 0}, def};
 
     // Redefinition is never allowed within a single FileDescriptorSet.
@@ -909,13 +980,13 @@ bool upb_symtab_add_defs(upb_symtab *s, upb_defs **defs, int num_defs,
 
     // Pass ownership from the deflist to the strtable.
     upb_strtable_insert(&tmptab, &e.e);
-    defs->defs[i] = NULL;
+    defs[i] = NULL;
   }
 
   // TODO: process the list of extensions by modifying entries from
   // tmptab in-place (copying them from the symtab first if necessary).
 
-  CHECK(upb_resolverefs(&tmptab, &s->symtab, status));
+  if (!upb_resolverefs(&tmptab, &s->symtab, status)) goto err;
 
   // The defs in tmptab have been vetted, and can be added to the symtab
   // without causing errors.  Now add all tmptab defs to the symtab,
@@ -946,6 +1017,7 @@ err:
     upb_def_unref(e->def);
   }
   upb_strtable_free(&tmptab);
+  for (int i = 0; i < num_defs; i++) upb_def_unref(defs[i]);
   return false;
 }
 
@@ -1026,20 +1098,18 @@ upb_def *upb_symtab_resolve(upb_symtab *s, upb_string *base, upb_string *symbol)
 
 void upb_symtab_addfds(upb_symtab *s, upb_src *src, upb_status *status)
 {
-  upb_defbuilder *b = upb_defbuilder_new();
-  upb_defbuilder_register_handlers(b, upb_src_gethandlers(src));
+  upb_defbuilder b;
+  upb_defbuilder_init(&b);
+  //upb_defbuilder_register_FileDescriptorSet(&b, upb_src_gethandlers(src));
+  upb_defbuilder_register_FileDescriptorSet(&b, NULL);
   if(!upb_src_run(src)) {
     upb_copyerr(status, upb_src_status(src));
+    upb_defbuilder_uninit(&b);
     return;
   }
-  upb_symtab_add_defs(s, b->defs, b->defs_len, false, status);
-  upb_deflist_uninit(&defs);
+  upb_symtab_add_defs(s, b.defs.defs, b.defs.len, false, status);
+  upb_defbuilder_uninit(&b);
   return;
-
-src_err:
-  upb_copyerr(status, upb_src_status(src));
-err:
-  upb_deflist_uninit(&defs);
 }
 
 
@@ -1074,8 +1144,10 @@ err:
 // complicated to support on big-endian machines.
 
 typedef struct {
+  upb_src src;
   upb_string *input;
   upb_strlen_t offset;
+  upb_dispatcher dispatcher;
 } upb_baredecoder;
 
 static uint64_t upb_baredecoder_readv64(upb_baredecoder *d)
@@ -1121,9 +1193,9 @@ bool upb_baredecoder_run(upb_baredecoder *d) {
   upb_dispatch_startmsg(&d->dispatcher);
   while(d->offset < upb_string_len(d->input)) {
     // Detect end-of-submessage.
-    while(d->offset >= *d->top) {
+    while(d->offset >= *top) {
       upb_dispatch_endsubmsg(&d->dispatcher);
-      d->offset = *(d->top--);
+      d->offset = *(top--);
     }
 
     uint32_t key = upb_baredecoder_readv64(d);
@@ -1134,16 +1206,16 @@ bool upb_baredecoder_run(upb_baredecoder *d) {
       uint32_t delim_len = upb_baredecoder_readv32(d);
       // We don't know if it's a string or a submessage; deliver first as
       // string.
-      str = upb_string_tryrecycle(str);
-      upb_string_substr(str, d->input, d->offset, d->delimited_len);
+      upb_string_recycle(&str);
+      upb_string_substr(str, d->input, d->offset, delim_len);
       upb_value v;
       upb_value_setstr(&v, str);
-      if(upb_dispatch_value(&d->dispatcher, &f, v) == UPB_TREAT_AS_SUBMSG) {
+      if(upb_dispatch_value(&d->dispatcher, &f, v) == BEGIN_SUBMSG) {
         // Should deliver as a submessage instead.
         upb_dispatch_startsubmsg(&d->dispatcher, &f);
-        *(++d->top) = d->offset + delimited_len;
+        *(++top) = d->offset + delim_len;
       } else {
-        d->offset += delimited_len;
+        d->offset += delim_len;
       }
     } else {
       upb_value v;
@@ -1167,23 +1239,24 @@ bool upb_baredecoder_run(upb_baredecoder *d) {
     }
   }
   upb_dispatch_endmsg(&d->dispatcher);
+  return true;
 }
 
-static upb_src_vtable upb_baredecoder_src_vtbl = {
-  (upb_src_getdef_fptr)&upb_baredecoder_getdef,
-  (upb_src_getval_fptr)&upb_baredecoder_getval,
-  (upb_src_getstr_fptr)&upb_baredecoder_getstr,
-  (upb_src_skipval_fptr)&upb_baredecoder_skipval,
-  (upb_src_startmsg_fptr)&upb_baredecoder_startmsg,
-  (upb_src_endmsg_fptr)&upb_baredecoder_endmsg,
-};
-
 static upb_baredecoder *upb_baredecoder_new(upb_string *str)
 {
+  //static upb_src_vtable vtbl = {
+  //  (upb_src_getdef_fptr)&upb_baredecoder_getdef,
+  //  (upb_src_getval_fptr)&upb_baredecoder_getval,
+  //  (upb_src_getstr_fptr)&upb_baredecoder_getstr,
+  //  (upb_src_skipval_fptr)&upb_baredecoder_skipval,
+  //  (upb_src_startmsg_fptr)&upb_baredecoder_startmsg,
+  //  (upb_src_endmsg_fptr)&upb_baredecoder_endmsg,
+  //};
   upb_baredecoder *d = malloc(sizeof(*d));
   d->input = upb_string_getref(str);
   d->offset = 0;
-  upb_src_init(&d->src, &upb_baredecoder_src_vtbl);
+  upb_dispatcher_init(&d->dispatcher);
+  //upb_src_init(&d->src, &vtbl);
   return d;
 }
 

core/upb_msg.c

@@ -7,6 +7,8 @@
  */
 
 #include "upb_msg.h"
+#include "upb_decoder.h"
+#include "upb_strstream.h"
 
 void _upb_elem_free(upb_value v, upb_fielddef *f) {
   switch(f->type) {
@@ -108,10 +110,13 @@ upb_value upb_field_tryrecycle(upb_valueptr p, upb_value val, upb_fielddef *f,
 
 void upb_msg_decodestr(upb_msg *msg, upb_msgdef *md, upb_string *str,
                        upb_status *status) {
-  (void)msg;
-  (void)md;
-  (void)str;
-  (void)status;
+  upb_stringsrc *ssrc = upb_stringsrc_new();
+  upb_stringsrc_reset(ssrc, str);
+  upb_decoder *d = upb_decoder_new(md);
+  upb_decoder_reset(d, upb_stringsrc_bytesrc(ssrc));
+
+  upb_decoder_free(d);
+  upb_stringsrc_free(ssrc);
 }
 
 void upb_msg_encodestr(upb_msg *msg, upb_msgdef *md, upb_string *str,

core/upb_stream.h

@@ -39,13 +39,16 @@ typedef enum {
   // Caller should continue sending values to the sink.
   UPB_CONTINUE,
 
-  // Skips to the end of the current submessage (or if we are at the top
-  // level, skips to the end of the entire message).
-  UPB_SKIP,
+  // An error occurred; check status for details.
+  UPB_ERROR,
 
-  // Caller should stop sending values; check sink status for details.
+  // Processing should stop for now, but could be resumed later.
   // If processing resumes later, it should resume with the next value.
-  UPB_STOP,
+  UPB_SUSPEND,
+
+  // Skips to the end of the current submessage (or if we are at the top
+  // level, skips to the end of the entire message).
+  UPB_SKIPSUBMSG,
 
   // When returned from a startsubmsg handler, indicates that the submessage
   // should be handled by a different set of handlers, which have been
@@ -117,6 +120,9 @@ INLINE void upb_handlers_uninit(upb_handlers *h);
 INLINE void upb_handlers_reset(upb_handlers *h);
 INLINE bool upb_handlers_isempty(upb_handlers *h);
 INLINE void upb_register_handlerset(upb_handlers *h, upb_handlerset *set);
+// TODO: for clients that want to increase efficiency by preventing bytesrcs
+// from automatically being converted to strings in the value callback.
+// INLINE void upb_handlers_use_bytesrcs(bool use_bytesrcs);
 INLINE void upb_set_handler_closure(upb_handlers *h, void *closure);
 
 // An object that transparently handles delegation so that the caller needs
@@ -140,21 +146,30 @@ INLINE upb_flow_t upb_dispatch_unknownval(upb_dispatcher *d,
 struct _upb_src;
 typedef struct _upb_src upb_src;
 
+bool upb_src_run(upb_src *src);
+upb_status *upb_src_status(upb_src *src);
 
-/* upb_bytesrc ****************************************************************/
-
-struct _upb_bytesrc;
-typedef struct _upb_bytesrc upb_bytesrc;
 
-// Returns the next string in the stream.  false is returned on error or eof.
-// The string must be at least "minlen" bytes long unless the stream is eof.
-INLINE bool upb_bytesrc_get(upb_bytesrc *src, upb_string *str, upb_strlen_t minlen);
+/* upb_bytesrc ****************************************************************/
 
-// Appends the next "len" bytes in the stream in-place to "str".  This should
-// be used when the caller needs to build a contiguous string of the existing
-// data in "str" with more data.  The call fails if fewer than len bytes are
-// available in the stream.
-INLINE bool upb_bytesrc_append(upb_bytesrc *src, upb_string *str, upb_strlen_t len);
+// Reads up to "count" bytes into "buf", returning the total number of bytes
+// read.  If <0, indicates error (check upb_bytesrc_status for details).
+INLINE upb_strlen_t upb_bytesrc_read(upb_bytesrc *src, void *buf,
+                                     upb_strlen_t count);
+
+// Like upb_bytesrc_read(), but modifies "str" in-place, possibly aliasing
+// existing string data (which avoids a copy).
+INLINE bool upb_bytesrc_getstr(upb_bytesrc *src, upb_string *str,
+                               upb_strlen_t count);
+
+// A convenience function for getting all the remaining data in a upb_bytesrc
+// as a upb_string.  Returns false and sets "status" if the operation fails.
+INLINE bool upb_bytesrc_getfullstr(upb_bytesrc *src, upb_string *str,
+                                   upb_status *status);
+INLINE bool upb_value_getfullstr(upb_value val, upb_string *str,
+                                 upb_status *status) {
+  return upb_bytesrc_getfullstr(upb_value_getbytesrc(val), str, status);
+}
 
 // Returns the current error status for the stream.
 // Note!  The "eof" flag works like feof() in C; it cannot report end-of-file
@@ -164,14 +179,21 @@ INLINE bool upb_bytesrc_append(upb_bytesrc *src, upb_string *str, upb_strlen_t l
 INLINE upb_status *upb_bytesrc_status(upb_bytesrc *src);
 INLINE bool upb_bytesrc_eof(upb_bytesrc *src);
 
+
 /* upb_bytesink ***************************************************************/
 
 struct _upb_bytesink;
 typedef struct _upb_bytesink upb_bytesink;
 
-// Puts the given string.  Returns the number of bytes that were actually,
-// consumed, which may be fewer than were in the string, or <0 on error.
-INLINE int32_t upb_bytesink_put(upb_bytesink *sink, upb_string *str);
+// Writes up to "count" bytes from "buf", returning the total number of bytes
+// written.  If <0, indicates error (check upb_bytesink_status() for details).
+INLINE upb_strlen_t upb_bytesink_write(upb_bytesink *sink, void *buf,
+                                       upb_strlen_t count);
+
+// Puts the given string, which may alias the string data (which avoids a
+// copy).  Returns the number of bytes that were actually, consumed, which may
+// be fewer than were in the string, or <0 on error.
+INLINE upb_strlen_t upb_bytesink_putstr(upb_bytesink *sink, upb_string *str);
 
 // Returns the current error status for the stream.
 INLINE upb_status *upb_bytesink_status(upb_bytesink *sink);

core/upb_stream_vtbl.h

@@ -20,23 +20,33 @@ extern "C" {
 
 // Typedefs for function pointers to all of the virtual functions.
 
+// upb_src
+struct _upb_src {
+};
+typedef struct {
+} upb_src_vtbl;
+
 // upb_bytesrc.
-typedef bool (*upb_bytesrc_get_fptr)(
-    upb_bytesrc *src, upb_string *str, upb_strlen_t minlen);
-typedef bool (*upb_bytesrc_append_fptr)(
-    upb_bytesrc *src, upb_string *str, upb_strlen_t len);
+typedef upb_strlen_t (*upb_bytesrc_read_fptr)(
+    upb_bytesrc *src, void *buf, upb_strlen_t count);
+typedef bool (*upb_bytesrc_getstr_fptr)(
+    upb_bytesrc *src, upb_string *str, upb_strlen_t count);
 
 // upb_bytesink.
-typedef int32_t (*upb_bytesink_put_fptr)(upb_bytesink *sink, upb_string *str);
+typedef upb_strlen_t (*upb_bytesink_write_fptr)(
+    upb_bytesink *bytesink, void *buf, upb_strlen_t count);
+typedef upb_strlen_t (*upb_bytesink_putstr_fptr)(
+    upb_bytesink *bytesink, upb_string *str);
 
 // Vtables for the above interfaces.
 typedef struct {
-  upb_bytesrc_get_fptr     get;
-  upb_bytesrc_append_fptr  append;
+  upb_bytesrc_read_fptr   read;
+  upb_bytesrc_getstr_fptr getstr;
 } upb_bytesrc_vtable;
 
 typedef struct {
-  upb_bytesink_put_fptr  put;
+  upb_bytesink_write_fptr  write;
+  upb_bytesink_putstr_fptr putstr;
 } upb_bytesink_vtable;
 
 // "Base Class" definitions; components that implement these interfaces should
@@ -69,19 +79,56 @@ INLINE void upb_bytesink_init(upb_bytesink *s, upb_bytesink_vtable *vtbl) {
 // Implementation of virtual function dispatch.
 
 // upb_bytesrc
-INLINE bool upb_bytesrc_get(
-    upb_bytesrc *bytesrc, upb_string *str, upb_strlen_t minlen) {
-  return bytesrc->vtbl->get(bytesrc, str, minlen);
-}
+INLINE upb_strlen_t upb_bytesrc_read(upb_bytesrc *src, void *buf,
+                                     upb_strlen_t count) {
+  return src->vtbl->read(src, buf, count);
+}
+
+INLINE bool upb_bytesrc_getstr(upb_bytesrc *src, upb_string *str,
+                               upb_strlen_t count) {
+  return src->vtbl->getstr(src, str, count);
+}
+
+INLINE bool upb_bytesrc_getfullstr(upb_bytesrc *src, upb_string *str,
+                                   upb_status *status) {
+  // We start with a getstr, because that could possibly alias data instead of
+  // copying.
+  if (!upb_bytesrc_getstr(src, str, UPB_STRLEN_MAX)) goto error;
+  // Trade-off between number of read calls and amount of overallocation.
+  const size_t bufsize = 4096;
+  while (!upb_bytesrc_eof(src)) {
+    upb_strlen_t len = upb_string_len(str);
+    char *buf = upb_string_getrwbuf(str, len + bufsize);
+    upb_strlen_t read = upb_bytesrc_read(src, buf + len, bufsize);
+    if (read < 0) goto error;
+    // Resize to proper size.
+    upb_string_getrwbuf(str, len + read);
+  }
+  return true;
 
-INLINE bool upb_bytesrc_append(
-    upb_bytesrc *bytesrc, upb_string *str, upb_strlen_t len) {
-  return bytesrc->vtbl->append(bytesrc, str, len);
+error:
+  upb_copyerr(status, upb_bytesrc_status(src));
+  return false;
 }
 
 INLINE upb_status *upb_bytesrc_status(upb_bytesrc *src) { return &src->status; }
 INLINE bool upb_bytesrc_eof(upb_bytesrc *src) { return src->eof; }
 
+
+// upb_bytesink
+INLINE upb_strlen_t upb_bytesink_write(upb_bytesink *sink, void *buf,
+                                       upb_strlen_t count) {
+  return sink->vtbl->write(sink, buf, count);
+}
+
+INLINE upb_strlen_t upb_bytesink_putstr(upb_bytesink *sink, upb_string *str) {
+  return sink->vtbl->putstr(sink, str);
+}
+
+INLINE upb_status *upb_bytesink_status(upb_bytesink *sink) {
+  return &sink->status;
+}
+
 // upb_handlers
 struct _upb_handlers {
   upb_handlerset *set;
@@ -182,17 +229,6 @@ INLINE upb_flow_t upb_dispatch_unknownval(upb_dispatcher *d,
                                           fieldnum, val);
 }
 
-// upb_bytesink
-INLINE int32_t upb_bytesink_put(upb_bytesink *sink, upb_string *str) {
-  return sink->vtbl->put(sink, str);
-}
-INLINE upb_status *upb_bytesink_status(upb_bytesink *sink) {
-  return &sink->status;
-}
-
-// upb_bytesink
-
-
 #ifdef __cplusplus
 }  /* extern "C" */
 #endif

core/upb_string.c

@@ -73,7 +73,7 @@ upb_string *upb_string_tryrecycle(upb_string *str) {
 
 char *upb_string_getrwbuf(upb_string *str, upb_strlen_t len) {
   // assert(str->ptr == NULL);
-  uint32_t size = upb_string_size(str);
+  upb_strlen_t size = upb_string_size(str);
   if (size < len) {
     size = upb_round_up_pow2(len);
     str->cached_mem = realloc(str->cached_mem, size);

core/upb_string.h

@@ -119,20 +119,21 @@ INLINE const char *upb_string_getrobuf(upb_string *str) { return str->ptr; }
 INLINE void upb_string_endread(upb_string *str) { (void)str; }
 
 // Attempts to recycle the string "str" so it may be reused and have different
-// data written to it.  The returned string is either "str" if it could be
-// recycled or a newly created string if "str" has other references.
+// data written to it.  After the function returns, "str" points to a writable
+// string, which is either the original string if it had no other references
+// or a newly created string if it did have other references.
 //
-// As a special case, passing NULL will allocate a new string.  This is
-// convenient for the pattern:
+// As a special case, passing a pointer to NULL will allocate a new string.
+// This is convenient for the pattern:
 //
 //   upb_string *str = NULL;
 //   while (x) {
 //     if (y) {
-//       str = upb_string_tryrecycle(str);
+//       upb_string_recycle(&str);
 //       upb_src_getstr(str);
 //     }
 //   }
-upb_string *upb_string_tryrecycle(upb_string *str);
+upb_string *upb_string_recycle(upb_string **str);
 
 // The options for setting the contents of a string.  These may only be called
 // when a string is first created or recycled; once other functions have been