Several changes to defs.

Biggest/key changes:
1. Defs are now nested per the .proto file syntax.
2. Options are parsed and vended.

upb/def.h

@@ -62,10 +62,16 @@ struct upb_fielddef;
 typedef struct upb_fielddef upb_fielddef;
 struct upb_filedef;
 typedef struct upb_filedef upb_filedef;
+struct upb_methoddef;
+typedef struct upb_methoddef upb_methoddef;
 struct upb_msgdef;
 typedef struct upb_msgdef upb_msgdef;
 struct upb_oneofdef;
 typedef struct upb_oneofdef upb_oneofdef;
+struct upb_servicedef;
+typedef struct upb_servicedef upb_servicedef;
+struct upb_streamdef;
+typedef struct upb_streamdef upb_streamdef;
 struct upb_symtab;
 typedef struct upb_symtab upb_symtab;
 
@@ -106,6 +112,8 @@ typedef enum {
  * protobuf wire format. */
 #define UPB_MAX_FIELDNUMBER ((1 << 29) - 1)
 
+const google_protobuf_FieldOptions *upb_fielddef_options(const upb_fielddef *f);
+bool upb_fielddef_hasoptions(const upb_fielddef *f);
 const char *upb_fielddef_fullname(const upb_fielddef *f);
 upb_fieldtype_t upb_fielddef_type(const upb_fielddef *f);
 upb_descriptortype_t upb_fielddef_descriptortype(const upb_fielddef *f);
@@ -126,6 +134,7 @@ bool upb_fielddef_isstring(const upb_fielddef *f);
 bool upb_fielddef_isseq(const upb_fielddef *f);
 bool upb_fielddef_isprimitive(const upb_fielddef *f);
 bool upb_fielddef_ismap(const upb_fielddef *f);
+bool upb_fielddef_hasdefault(const upb_fielddef *f);
 int64_t upb_fielddef_defaultint64(const upb_fielddef *f);
 int32_t upb_fielddef_defaultint32(const upb_fielddef *f);
 uint64_t upb_fielddef_defaultuint64(const upb_fielddef *f);
@@ -140,11 +149,14 @@ const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f);
 const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f);
 const upb_msglayout_field *upb_fielddef_layout(const upb_fielddef *f);
 const upb_msglayout_ext *_upb_fielddef_extlayout(const upb_fielddef *f);
+bool _upb_fielddef_proto3optional(const upb_fielddef *f);
 
 /* upb_oneofdef ***************************************************************/
 
 typedef upb_inttable_iter upb_oneof_iter;
 
+const google_protobuf_OneofOptions *upb_oneofdef_options(const upb_oneofdef *o);
+bool upb_oneofdef_hasoptions(const upb_oneofdef *o);
 const char *upb_oneofdef_name(const upb_oneofdef *o);
 const upb_msgdef *upb_oneofdef_containingtype(const upb_oneofdef *o);
 uint32_t upb_oneofdef_index(const upb_oneofdef *o);
@@ -196,11 +208,13 @@ typedef upb_strtable_iter upb_msg_oneof_iter;
 #define UPB_TIMESTAMP_SECONDS 1
 #define UPB_TIMESTAMP_NANOS 2
 
+const google_protobuf_MessageOptions *upb_msgdef_options(const upb_msgdef *m);
+bool upb_msgdef_hasoptions(const upb_msgdef *m);
 const char *upb_msgdef_fullname(const upb_msgdef *m);
 const upb_filedef *upb_msgdef_file(const upb_msgdef *m);
+const upb_msgdef *upb_msgdef_containingtype(const upb_msgdef *m);
 const char *upb_msgdef_name(const upb_msgdef *m);
 upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m);
-bool upb_msgdef_mapentry(const upb_msgdef *m);
 upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m);
 bool upb_msgdef_iswrapper(const upb_msgdef *m);
 bool upb_msgdef_isnumberwrapper(const upb_msgdef *m);
@@ -227,6 +241,18 @@ UPB_INLINE const upb_fielddef *upb_msgdef_ntofz(const upb_msgdef *m,
   return upb_msgdef_ntof(m, name, strlen(name));
 }
 
+UPB_INLINE bool upb_msgdef_mapentry(const upb_msgdef *m) {
+  return google_protobuf_MessageOptions_map_entry(upb_msgdef_options(m));
+}
+
+/* Nested entities. */
+int upb_msgdef_nestedmsgcount(const upb_msgdef *m);
+int upb_msgdef_nestedenumcount(const upb_msgdef *m);
+int upb_msgdef_nestedextcount(const upb_msgdef *m);
+const upb_msgdef *upb_msgdef_nestedmsg(const upb_msgdef *m, int i);
+const upb_enumdef *upb_msgdef_nestedenum(const upb_msgdef *m, int i);
+const upb_fielddef *upb_msgdef_nestedext(const upb_msgdef *m, int i);
+
 /* Lookup of either field or oneof by name.  Returns whether either was found.
  * If the return is true, then the found def will be set, and the non-found
  * one set to NULL. */
@@ -242,6 +268,10 @@ UPB_INLINE bool upb_msgdef_lookupnamez(const upb_msgdef *m, const char *name,
 /* Returns a field by either JSON name or regular proto name. */
 const upb_fielddef *upb_msgdef_lookupjsonname(const upb_msgdef *m,
                                               const char *name, size_t len);
+UPB_INLINE const upb_fielddef *upb_msgdef_lookupjsonnamez(const upb_msgdef *m,
+                                                          const char *name) {
+  return upb_msgdef_lookupjsonname(m, name, strlen(name));
+}
 
 /* DEPRECATED, slated for removal */
 int upb_msgdef_numfields(const upb_msgdef *m);
@@ -275,9 +305,12 @@ int32_t upb_extrange_end(const upb_extrange *r);
 
 typedef upb_strtable_iter upb_enum_iter;
 
+const google_protobuf_EnumOptions *upb_enumdef_options(const upb_enumdef *e);
+bool upb_enumdef_hasoptions(const upb_enumdef *e);
 const char *upb_enumdef_fullname(const upb_enumdef *e);
 const char *upb_enumdef_name(const upb_enumdef *e);
 const upb_filedef *upb_enumdef_file(const upb_enumdef *e);
+const upb_msgdef *upb_enumdef_containingtype(const upb_enumdef *e);
 int32_t upb_enumdef_default(const upb_enumdef *e);
 int upb_enumdef_valuecount(const upb_enumdef *e);
 const upb_enumvaldef *upb_enumdef_value(const upb_enumdef *e, int i);
@@ -303,6 +336,9 @@ UPB_INLINE const upb_enumvaldef *upb_enumdef_lookupnamez(const upb_enumdef *e,
 
 /* upb_enumvaldef *************************************************************/
 
+const google_protobuf_EnumValueOptions *upb_enumvaldef_options(
+    const upb_enumvaldef *e);
+bool upb_enumvaldef_hasoptions(const upb_enumvaldef *e);
 const char *upb_enumvaldef_fullname(const upb_enumvaldef *e);
 const char *upb_enumvaldef_name(const upb_enumvaldef *e);
 int32_t upb_enumvaldef_number(const upb_enumvaldef *e);
@@ -310,18 +346,54 @@ const upb_enumdef *upb_enumvaldef_enum(const upb_enumvaldef *e);
 
 /* upb_filedef ****************************************************************/
 
+const google_protobuf_FileOptions *upb_filedef_options(const upb_filedef *f);
+bool upb_filedef_hasoptions(const upb_filedef *f);
 const char *upb_filedef_name(const upb_filedef *f);
 const char *upb_filedef_package(const upb_filedef *f);
 const char *upb_filedef_phpprefix(const upb_filedef *f);
 const char *upb_filedef_phpnamespace(const upb_filedef *f);
 upb_syntax_t upb_filedef_syntax(const upb_filedef *f);
 int upb_filedef_depcount(const upb_filedef *f);
-int upb_filedef_msgcount(const upb_filedef *f);
-int upb_filedef_enumcount(const upb_filedef *f);
+int upb_filedef_publicdepcount(const upb_filedef *f);
+int upb_filedef_toplvlmsgcount(const upb_filedef *f);
+int upb_filedef_toplvlenumcount(const upb_filedef *f);
+int upb_filedef_toplvlextcount(const upb_filedef *f);
+int upb_filedef_servicecount(const upb_filedef *f);
 const upb_filedef *upb_filedef_dep(const upb_filedef *f, int i);
-const upb_msgdef *upb_filedef_msg(const upb_filedef *f, int i);
-const upb_enumdef *upb_filedef_enum(const upb_filedef *f, int i);
+const upb_filedef *upb_filedef_publicdep(const upb_filedef *f, int i);
+const upb_msgdef *upb_filedef_toplvlmsg(const upb_filedef *f, int i);
+const upb_enumdef *upb_filedef_toplvlenum(const upb_filedef *f, int i);
+const upb_fielddef *upb_filedef_toplvlext(const upb_filedef *f, int i);
+const upb_servicedef *upb_filedef_service(const upb_filedef *f, int i);
 const upb_symtab *upb_filedef_symtab(const upb_filedef *f);
+const int32_t *_upb_filedef_publicdepnums(const upb_filedef *f);
+
+/* upb_methoddef **************************************************************/
+
+const google_protobuf_MethodOptions *upb_methoddef_options(
+    const upb_methoddef *m);
+bool upb_methoddef_hasoptions(const upb_methoddef *m);
+const char *upb_methoddef_fullname(const upb_methoddef *m);
+const char *upb_methoddef_name(const upb_methoddef *m);
+const upb_servicedef *upb_methoddef_service(const upb_methoddef *m);
+const upb_msgdef *upb_methoddef_inputtype(const upb_methoddef *m);
+const upb_msgdef *upb_methoddef_outputtype(const upb_methoddef *m);
+bool upb_methoddef_clientstreaming(const upb_methoddef *m);
+bool upb_methoddef_serverstreaming(const upb_methoddef *m);
+
+/* upb_servicedef *************************************************************/
+
+const google_protobuf_ServiceOptions *upb_servicedef_options(
+    const upb_servicedef *s);
+bool upb_servicedef_hasoptions(const upb_servicedef *s);
+const char *upb_servicedef_fullname(const upb_servicedef *s);
+const char *upb_servicedef_name(const upb_servicedef *s);
+int upb_servicedef_index(const upb_servicedef *s);
+const upb_filedef *upb_servicedef_file(const upb_servicedef *s);
+int upb_servicedef_methodcount(const upb_servicedef *s);
+const upb_methoddef *upb_servicedef_method(const upb_servicedef *s, int i);
+const upb_methoddef *upb_servicedef_lookupmethod(const upb_servicedef *s,
+                                                 const char *name);
 
 /* upb_symtab *****************************************************************/
 
@@ -337,9 +409,12 @@ const upb_fielddef *upb_symtab_lookupext(const upb_symtab *s, const char *sym);
 const upb_fielddef *upb_symtab_lookupext2(const upb_symtab *s, const char *sym,
                                          size_t len);
 const upb_filedef *upb_symtab_lookupfile(const upb_symtab *s, const char *name);
+const upb_servicedef *upb_symtab_lookupservice(const upb_symtab *s,
+                                               const char *name);
+const upb_filedef *upb_symtab_lookupfileforsym(const upb_symtab *s,
+                                               const char *name);
 const upb_filedef *upb_symtab_lookupfile2(
     const upb_symtab *s, const char *name, size_t len);
-int upb_symtab_filecount(const upb_symtab *s);
 const upb_filedef *upb_symtab_addfile(
     upb_symtab *s, const google_protobuf_FileDescriptorProto *file,
     upb_status *status);
@@ -347,7 +422,12 @@ size_t _upb_symtab_bytesloaded(const upb_symtab *s);
 upb_arena *_upb_symtab_arena(const upb_symtab *s);
 const upb_fielddef *_upb_symtab_lookupextfield(const upb_symtab *s,
                                                const upb_msglayout_ext *ext);
+const upb_fielddef *upb_symtab_lookupextbynum(const upb_symtab *s,
+                                              const upb_msgdef *m,
+                                              int32_t fieldnum);
 const upb_extreg *upb_symtab_extreg(const upb_symtab *s);
+const upb_fielddef **upb_symtab_getallexts(const upb_symtab *s,
+                                           const upb_msgdef *m, size_t *count);
 
 /* For generated code only: loads a generated descriptor. */
 typedef struct upb_def_init {
@@ -358,6 +438,7 @@ typedef struct upb_def_init {
 } upb_def_init;
 
 bool _upb_symtab_loaddefinit(upb_symtab *s, const upb_def_init *init);
+void _upb_symtab_allownameconflicts(upb_symtab *s);
 
 #include "upb/port_undef.inc"
 

upb/table.c

@@ -805,6 +805,100 @@ void upb_inttable_next(upb_inttable_iter *iter) {
   }
 }
 
+bool upb_inttable_next2(const upb_inttable *t, uintptr_t *key, upb_value *val,
+                        intptr_t *iter) {
+  intptr_t i = *iter;
+  if (i < t->array_size) {
+    while (++i < t->array_size) {
+      upb_tabval ent = t->array[i];
+      if (upb_arrhas(ent)) {
+        *key = i;
+        *val = _upb_value_val(ent.val);
+        *iter = i;
+        return true;
+      }
+    }
+  }
+
+  size_t tab_idx = next(&t->t, i == -1 ? -1 : i - t->array_size);
+  if (tab_idx < upb_table_size(&t->t)) {
+    upb_tabent *ent = &t->t.entries[tab_idx];
+    *key = ent->key;
+    *val = _upb_value_val(ent->val.val);
+    *iter = tab_idx + t->array_size;
+    return true;
+  }
+
+  return false;
+}
+
+void upb_inttable_removeiter(upb_inttable *t, intptr_t *iter) {
+  intptr_t i = *iter;
+  if (i < t->array_size) {
+    t->array_count--;
+    mutable_array(t)[i].val = -1;
+  } else {
+    upb_tabent *ent = &t->t.entries[i - t->array_size];
+    upb_tabent *prev = NULL;
+
+    // Linear search, not great.
+    upb_tabent *end = &t->t.entries[upb_table_size(&t->t)];
+    for (upb_tabent *e = t->t.entries; e != end; e++) {
+      if (e->next == ent) {
+        prev = e;
+        break;
+      }
+    }
+
+    if (prev) {
+      prev->next = ent->next;
+    }
+
+    t->t.count--;
+    ent->key = 0;
+    ent->next = NULL;
+  }
+}
+
+bool upb_strtable_next2(const upb_strtable *t, upb_strview *key, upb_value *val,
+                        intptr_t *iter) {
+  size_t tab_idx = next(&t->t, *iter);
+  if (tab_idx < upb_table_size(&t->t)) {
+    upb_tabent *ent = &t->t.entries[tab_idx];
+    uint32_t len;
+    key->data = upb_tabstr(ent->key, &len);
+    key->size = len;
+    *val = _upb_value_val(ent->val.val);
+    *iter = tab_idx;
+    return true;
+  }
+
+  return false;
+}
+
+void upb_strtable_removeiter(upb_strtable *t, intptr_t *iter) {
+  intptr_t i = *iter;
+  upb_tabent *ent = &t->t.entries[i];
+  upb_tabent *prev = NULL;
+
+  // Linear search, not great.
+  upb_tabent *end = &t->t.entries[upb_table_size(&t->t)];
+  for (upb_tabent *e = t->t.entries; e != end; e++) {
+    if (e->next == ent) {
+      prev = e;
+      break;
+    }
+  }
+
+  if (prev) {
+    prev->next = ent->next;
+  }
+
+  t->t.count--;
+  ent->key = 0;
+  ent->next = NULL;
+}
+
 bool upb_inttable_done(const upb_inttable_iter *i) {
   if (!i->t) return true;
   if (i->array_part) {

upb/table_internal.h

@@ -263,6 +263,38 @@ bool upb_strtable_resize(upb_strtable *t, size_t size_lg2, upb_arena *a);
 
 /* Iterators ******************************************************************/
 
+/* New-style iterators.  Much simpler, iterator state is held in size_t.
+ *
+ *   intptr_t iter = UPB_INTTABLE_BEGIN;
+ *   uintptr_t key;
+ *   upb_value val;
+ *   while (upb_inttable_next2(t, &key, &val, &iter)) {
+ *      // ...
+ *   }
+ */
+
+#define UPB_INTTABLE_BEGIN -1
+
+bool upb_inttable_next2(const upb_inttable *t, uintptr_t *key, upb_value *val,
+                        intptr_t *iter);
+void upb_inttable_removeiter(upb_inttable *t, intptr_t *iter);
+
+/* New-style iterators.  Much simpler, iterator state is held in size_t.
+ *
+ *   intptr_t iter = UPB_INTTABLE_BEGIN;
+ *   upb_strview key;
+ *   upb_value val;
+ *   while (upb_strtable_next2(t, &key, &val, &iter)) {
+ *      // ...
+ *   }
+ */
+
+#define UPB_STRTABLE_BEGIN -1
+
+bool upb_strtable_next2(const upb_strtable *t, upb_strview *key, upb_value *val,
+                        intptr_t *iter);
+void upb_strtable_removeiter(upb_strtable *t, intptr_t *iter);
+
 /* Iterators for int and string tables.  We are subject to some kind of unusual
  * design constraints:
  *

upbc/protoc-gen-upb.cc

@@ -138,12 +138,12 @@ void AddExtensionsFromMessage(
 std::vector<const protobuf::FieldDescriptor*> SortedExtensions(
     const protobuf::FileDescriptor* file) {
   std::vector<const protobuf::FieldDescriptor*> ret;
-  for (int i = 0; i < file->message_type_count(); i++) {
-    AddExtensionsFromMessage(file->message_type(i), &ret);
-  }
   for (int i = 0; i < file->extension_count(); i++) {
     ret.push_back(file->extension(i));
   }
+  for (int i = 0; i < file->message_type_count(); i++) {
+    AddExtensionsFromMessage(file->message_type(i), &ret);
+  }
   return ret;
 }
 
@@ -729,8 +729,42 @@ void WriteHeader(const protobuf::FileDescriptor* file, Output& output) {
     GenerateExtensionInHeader(ext, output);
   }
 
+
   output("extern const upb_msglayout_file $0;\n\n", FileLayoutName(file));
 
+  if (file->name() == protobuf::FileDescriptorProto::descriptor()->file()->name()) {
+    // This is gratuitously inefficient with how many times it rebuilds
+    // MessageLayout objects for the same message. But we only do this for one
+    // proto (descriptor.proto) so we don't worry about it.
+    const protobuf::Descriptor* max32 = nullptr;
+    const protobuf::Descriptor* max64 = nullptr;
+    for (auto message : this_file_messages) {
+      if (absl::EndsWith(message->name(), "Options")) {
+        MessageLayout layout(message);
+        if (max32 == nullptr) {
+          max32 = message;
+          max64 = message;
+        } else {
+          if (layout.message_size().size32 >
+              MessageLayout(max32).message_size().size32) {
+            max32 = message;
+          }
+          if (layout.message_size().size64 >
+              MessageLayout(max64).message_size().size64) {
+            max64 = message;
+          }
+        }
+      }
+    }
+
+    output("/* Max size 32 is $0 */\n", max32->full_name());
+    output("/* Max size 64 is $0 */\n", max64->full_name());
+    MessageLayout::Size size;
+    size.size32 = MessageLayout(max32).message_size().size32;
+    size.size64 = MessageLayout(max32).message_size().size64;
+    output("#define _UPB_MAXOPT_SIZE $0\n\n", GetSizeInit(size));
+  }
+
   output(
       "#ifdef __cplusplus\n"
       "}  /* extern \"C\" */\n"