Fixes for PHP. (#286)

- A new PHP-specific upb amalgamation. It contains everything related to upb_msg, but leaves out all of the old handlers-related interfaces and encoders/decoders. # Schema/Defs Changes - Changed `upb_fielddef_msgsubdef()` and `upb_fielddef_enumsubdef()` to return `NULL` instead of assert-failing if the field is not a message or enum. - Added `upb_msgdef_iswrapper()`, to test whether this is a wrapper well-known type. # Decoder - Decoder bugfix: when we parse a submessage inside a oneof, we need to clear out any previous data, so we don't misinterpret it as a pointer to an existing submessage. # JSON Decoder - Allowed well-known types at the top level to have their special processing. - Fixed a bug that could occur when parsing nested empty lists/objects, eg `[[]]`. - Made the "ignore unknown" option also be permissive about unknown enumerators by setting them to 0. # JSON Encoder - Allowed well-known types at the top level to have their special processing. - Removed all spaces after `:` and `,` characters, to match the old encoder and pass goldenfile tests. # Message / Reflection - Changed `upb_msg_hasoneof()` -> `upb_msg_whichoneof()`. The new function returns the `upb_fielddef*` of whichever oneof is set. - Implemented `upb_msg_clearfield()` and added/implemented `upb_msg_clear()`. - Added `upb_msg_discardunknown()`. Part of me thinks this should go in a util library instead of core reflection since it is a recursive algorithm. # Compiler - Always emit descriptors as an array instead of as a string, to avoid exceeding maximum string lengths. If this becomes a speed issue later we can go back to two separate paths.
5 years ago · 543a0ce8f2
parent 93e2a40881
commit 543a0ce8f2
13 changed files with 223 additions and 125 deletions
--- a/63
+++ b/63
@ -120,7 +120,13 @@ cc_library(
 )

 upb_proto_library(
-    name = "descriptor_upbproto",
+    name = "descriptor_upb_proto",
+    visibility = ["//visibility:public"],
+    deps = ["@com_google_protobuf//:descriptor_proto"],
+)
+
+upb_proto_reflection_library(
+    name = "descriptor_upb_proto_reflection",
    visibility = ["//visibility:public"],
    deps = ["@com_google_protobuf//:descriptor_proto"],
 )
@ -143,7 +149,7 @@ cc_library(
    }),
    visibility = ["//visibility:public"],
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":port",
        ":table",
        ":upb",
@ -239,7 +245,7 @@ cc_library(
        "//conditions:default": COPTS,
    }),
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":handlers",
        ":port",
        ":reflection",
@ -276,7 +282,7 @@ cc_library(
        "upb/bindings/stdc++/string.h",
    ],
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":handlers",
        ":port",
        ":upb",
@ -335,7 +341,7 @@ cc_binary(
    testonly = 1,
    srcs = ["tests/benchmark.cc"],
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":descriptor_upbreflection",
        "@com_github_google_benchmark//:benchmark_main",
    ],
@ -387,7 +393,7 @@ proto_library(
 )

 upb_proto_library(
-    name = "test_upbproto",
+    name = "test_upb_proto",
    testonly = 1,
    deps = [":test_proto"],
 )
@ -398,7 +404,7 @@ cc_test(
    deps = [
        ":test_messages_proto3_proto_upb",
        ":empty_upbdefs_proto",
-        ":test_upbproto",
+        ":test_upb_proto",
        ":upb_test",
    ],
 )
@ -428,7 +434,7 @@ proto_library(
 )

 upb_proto_reflection_library(
-    name = "test_decoder_upbproto",
+    name = "test_decoder_upb_proto",
    deps = [":test_decoder_proto"],
 )

@ -445,7 +451,7 @@ cc_test(
    deps = [
        ":handlers",
        ":port",
-        ":test_decoder_upbproto",
+        ":test_decoder_upb_proto",
        ":upb",
        ":upb_pb",
        ":upb_test",
@ -460,7 +466,7 @@ proto_library(
 )

 upb_proto_reflection_library(
-    name = "test_cpp_upbproto",
+    name = "test_cpp_upb_proto",
    deps = ["test_cpp_proto"],
 )

@ -475,7 +481,7 @@ cc_test(
        ":handlers",
        ":port",
        ":reflection",
-        ":test_cpp_upbproto",
+        ":test_cpp_upb_proto",
        ":upb",
        ":upb_pb",
        ":upb_test",
@ -514,7 +520,7 @@ cc_binary(
        ":fuzz": ["HAVE_FUZZER"],
    }),
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":upb",
    ],
 )
@ -528,7 +534,7 @@ cc_test(
        "//conditions:default": CPPOPTS,
    }),
    deps = [
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":descriptor_upbreflection",
        ":upb",
        ":upb_cc_bindings",
@ -549,17 +555,17 @@ proto_library(
 )

 upb_proto_reflection_library(
-    name = "test_json_upbprotoreflection",
+    name = "test_json_upb_proto_reflection",
    deps = ["test_json_proto"],
 )

 upb_proto_library(
-    name = "test_json_enum_from_separate_upbproto",
+    name = "test_json_enum_from_separate_upb_proto",
    deps = [":test_json_enum_from_separate"],
 )

 upb_proto_library(
-    name = "test_json_upbproto",
+    name = "test_json_upb_proto",
    deps = [":test_json_proto"],
 )

@ -573,8 +579,8 @@ cc_test(
        "//conditions:default": CPPOPTS,
    }),
    deps = [
-        ":test_json_upbproto",
-        ":test_json_upbprotoreflection",
+        ":test_json_upb_proto",
+        ":test_json_upb_proto_reflection",
        ":upb_json",
        ":upb_test",
    ],
@ -662,7 +668,7 @@ upb_amalgamation(
    amalgamator = ":amalgamate",
    libs = [
        ":upb",
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
        ":reflection",
        ":handlers",
        ":port",
@ -682,16 +688,17 @@ cc_library(
 )

 upb_amalgamation(
-    name = "gen_core_amalgamation",
-    prefix = "core-",
+    name = "gen_php_amalgamation",
+    prefix = "php-",
    outs = [
-        "core-upb.c",
-        "core-upb.h",
+        "php-upb.c",
+        "php-upb.h",
    ],
    amalgamator = ":amalgamate",
    libs = [
        ":upb",
-        ":descriptor_upbproto",
+        ":descriptor_upb_proto",
+        ":descriptor_upb_proto_reflection",
        ":reflection",
        ":port",
        ":json",
@ -699,9 +706,9 @@ upb_amalgamation(
 )

 cc_library(
-    name = "core_amalgamation",
-    srcs = ["core-upb.c"],
-    hdrs = ["core-upb.h"],
+    name = "php_amalgamation",
+    srcs = ["php-upb.c"],
+    hdrs = ["php-upb.h"],
    copts = select({
        ":windows": [],
        "//conditions:default": COPTS,
@ -851,7 +858,7 @@ genrule(

 genrule(
    name = "copy_protos",
-    srcs = [":descriptor_upbproto"],
+    srcs = [":descriptor_upb_proto"],
    outs = [
        "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.c",
        "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.h",
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -88,7 +88,7 @@ add_library(reflection
  upb/def.hpp
  upb/reflection.h)
 target_link_libraries(reflection
-  descriptor_upbproto
+  descriptor_upb_proto
  port
  table
  upb)
@ -134,7 +134,7 @@ add_library(upb_pb
  upb/pb/encoder.h
  upb/pb/textprinter.h)
 target_link_libraries(upb_pb
-  descriptor_upbproto
+  descriptor_upb_proto
  handlers
  port
  reflection
@ -150,7 +150,7 @@ target_link_libraries(upb_json
  upb_pb)
 add_library(upb_cc_bindings INTERFACE)
 target_link_libraries(upb_cc_bindings INTERFACE
-  descriptor_upbproto
+  descriptor_upb_proto
  handlers
  port
  upb)
--- a/tools/make_cmakelists.py
+++ b/tools/make_cmakelists.py
@ -36,9 +36,11 @@ class BuildFileFunctions(object):
    pass

  def cc_library(self, **kwargs):
-    if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator":
+    if kwargs["name"].endswith("amalgamation"):
      return
-    if kwargs["name"] == "core_amalgamation" or kwargs["name"] == "lupb":
+    if kwargs["name"] == "upbc_generator":
+      return
+    if kwargs["name"] == "lupb":
      return
    files = kwargs.get("srcs", []) + kwargs.get("hdrs", [])
    found_files = []
--- a/upb/decode.c
+++ b/upb/decode.c
@ -408,7 +408,11 @@ static const char *decode_tomsg(upb_decstate *d, const char *ptr, upb_msg *msg,
  /* Set presence if necessary. */
  if (field->presence < 0) {
    /* Oneof case */
-    *UPB_PTR_AT(msg, -field->presence, int32_t) = field->number;
+    int32_t *oneof_case = UPB_PTR_AT(msg, -field->presence, int32_t);
+    if (op == OP_SUBMSG && *oneof_case != field->number) {
+      memset(mem, 0, sizeof(void*));
+    }
+    *oneof_case = field->number;
  } else if (field->presence > 0) {
    /* Hasbit */
    uint32_t hasbit = field->presence;
--- a/upb/def.c
+++ b/upb/def.c
@ -577,13 +577,11 @@ const char *upb_fielddef_defaultstr(const upb_fielddef *f, size_t *len) {
 }

 const upb_msgdef *upb_fielddef_msgsubdef(const upb_fielddef *f) {
-  UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_MESSAGE);
-  return f->sub.msgdef;
+  return upb_fielddef_type(f) == UPB_TYPE_MESSAGE ? f->sub.msgdef : NULL;
 }

 const upb_enumdef *upb_fielddef_enumsubdef(const upb_fielddef *f) {
-  UPB_ASSERT(upb_fielddef_type(f) == UPB_TYPE_ENUM);
-  return f->sub.enumdef;
+  return upb_fielddef_type(f) == UPB_TYPE_ENUM ? f->sub.enumdef : NULL;
 }

 const upb_msglayout_field *upb_fielddef_layout(const upb_fielddef *f) {
@ -751,6 +749,12 @@ bool upb_msgdef_isnumberwrapper(const upb_msgdef *m) {
         type <= UPB_WELLKNOWN_UINT32VALUE;
 }

+bool upb_msgdef_iswrapper(const upb_msgdef *m) {
+  upb_wellknowntype_t type = upb_msgdef_wellknowntype(m);
+  return type >= UPB_WELLKNOWN_DOUBLEVALUE &&
+         type <= UPB_WELLKNOWN_BOOLVALUE;
+}
+
 void upb_msg_field_begin(upb_msg_field_iter *iter, const upb_msgdef *m) {
  upb_inttable_begin(iter, &m->itof);
 }
--- a/upb/def.h
+++ b/upb/def.h
@ -176,6 +176,7 @@ int upb_msgdef_numrealoneofs(const upb_msgdef *m);
 upb_syntax_t upb_msgdef_syntax(const upb_msgdef *m);
 bool upb_msgdef_mapentry(const upb_msgdef *m);
 upb_wellknowntype_t upb_msgdef_wellknowntype(const upb_msgdef *m);
+bool upb_msgdef_iswrapper(const upb_msgdef *m);
 bool upb_msgdef_isnumberwrapper(const upb_msgdef *m);
 const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i);
 const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name,
--- a/upb/json_decode.c
+++ b/upb/json_decode.c
@ -157,15 +157,11 @@ static void jsondec_push(jsondec *d) {
 }

 static bool jsondec_seqnext(jsondec *d, char end_ch) {
+  bool is_first = d->is_first;
+  d->is_first = false;
  jsondec_skipws(d);
  if (*d->ptr == end_ch) return false;
-
-  if (d->is_first) {
-    d->is_first = false;
-  } else {
-    jsondec_parselit(d, ",");
-  }
-
+  if (!is_first) jsondec_parselit(d, ",");
  return true;
 }

@ -772,7 +768,12 @@ static upb_msgval jsondec_enum(jsondec *d, const upb_fielddef *f) {
    upb_strview str = jsondec_string(d);
    upb_msgval val;
    if (!upb_enumdef_ntoi(e, str.data, str.size, &val.int32_val)) {
-      jsondec_err(d, "Unknown enumerator");
+      if (d->options & UPB_JSONDEC_IGNOREUNKNOWN) {
+        val.int32_val = 0;
+      } else {
+        jsondec_errf(d, "Unknown enumerator: '" UPB_STRVIEW_FORMAT "'",
+                     UPB_STRVIEW_ARGS(str));
+      }
    }
    return val;
  } else {
@ -877,14 +878,15 @@ static void jsondec_field(jsondec *d, upb_msg *msg, const upb_msgdef *m) {

  if (!f) {
    if ((d->options & UPB_JSONDEC_IGNOREUNKNOWN) == 0) {
-      jsondec_err(d, "Unknown field");
+      jsondec_errf(d, "Unknown field: '" UPB_STRVIEW_FORMAT "'",
+                   UPB_STRVIEW_ARGS(name));
    }
    jsondec_skipval(d);
    return;
  }

  if (upb_fielddef_containingoneof(f) &&
-      upb_msg_hasoneof(msg, upb_fielddef_containingoneof(f))) {
+      upb_msg_whichoneof(msg, upb_fielddef_containingoneof(f))) {
    jsondec_err(d, "More than one field for this oneof.");
  }

@ -915,7 +917,9 @@ static void jsondec_field(jsondec *d, upb_msg *msg, const upb_msgdef *m) {

 static void jsondec_object(jsondec *d, upb_msg *msg, const upb_msgdef *m) {
  jsondec_objstart(d);
-  while (jsondec_objnext(d)) jsondec_field(d, msg, m);
+  while (jsondec_objnext(d)) {
+    jsondec_field(d, msg, m);
+  }
  jsondec_objend(d);
 }

@ -1401,6 +1405,6 @@ bool upb_json_decode(const char *buf, size_t size, upb_msg *msg,

  if (setjmp(d.err)) return false;

-  jsondec_object(&d, msg, m);
+  jsondec_tomsg(&d, msg, m);
  return true;
 }
--- a/upb/json_encode.c
+++ b/upb/json_encode.c
@ -318,16 +318,16 @@ static void jsonenc_any(jsonenc *e, const upb_msg *msg, const upb_msgdef *m) {
    jsonenc_err(e, "Error decoding message in Any");
  }

-  jsonenc_putstr(e, "{\"@type\": ");
+  jsonenc_putstr(e, "{\"@type\":");
  jsonenc_string(e, type_url);
-  jsonenc_putstr(e, ", ");
+  jsonenc_putstr(e, ",");

  if (upb_msgdef_wellknowntype(any_m) == UPB_WELLKNOWN_UNSPECIFIED) {
-    /* Regular messages: {"@type": "...", "foo": 1, "bar": 2} */
+    /* Regular messages: {"@type": "...","foo": 1, "bar": 2} */
    jsonenc_msgfields(e, any, any_m);
  } else {
-    /* Well-known type: {"@type": "...", "value": <well-known encoding>} */
-    jsonenc_putstr(e, "value: ");
+    /* Well-known type: {"@type": "...","value": <well-known encoding>} */
+    jsonenc_putstr(e, "\"value\":");
    jsonenc_msgfield(e, any, any_m);
  }

@ -393,14 +393,16 @@ static void jsonenc_struct(jsonenc *e, const upb_msg *msg,

  jsonenc_putstr(e, "{");

-  while (upb_mapiter_next(fields, &iter)) {
-    upb_msgval key = upb_mapiter_key(fields, iter);
-    upb_msgval val = upb_mapiter_value(fields, iter);
+  if (fields) {
+    while (upb_mapiter_next(fields, &iter)) {
+      upb_msgval key = upb_mapiter_key(fields, iter);
+      upb_msgval val = upb_mapiter_value(fields, iter);

-    jsonenc_putsep(e, ", ", &first);
-    jsonenc_string(e, key.str_val);
-    jsonenc_putstr(e, ": ");
-    jsonenc_value(e, val.msg_val, upb_fielddef_msgsubdef(value_f));
+      jsonenc_putsep(e, ",", &first);
+      jsonenc_string(e, key.str_val);
+      jsonenc_putstr(e, ":");
+      jsonenc_value(e, val.msg_val, upb_fielddef_msgsubdef(value_f));
+    }
  }

  jsonenc_putstr(e, "}");
@ -411,17 +413,19 @@ static void jsonenc_listvalue(jsonenc *e, const upb_msg *msg,
  const upb_fielddef *values_f = upb_msgdef_itof(m, 1);
  const upb_msgdef *values_m = upb_fielddef_msgsubdef(values_f);
  const upb_array *values = upb_msg_get(msg, values_f).array_val;
-  const size_t size = upb_array_size(values);
  size_t i;
  bool first = true;

  jsonenc_putstr(e, "[");

-  for (i = 0; i < size; i++) {
-    upb_msgval elem = upb_array_get(values, i);
+  if (values) {
+    const size_t size = upb_array_size(values);
+    for (i = 0; i < size; i++) {
+      upb_msgval elem = upb_array_get(values, i);

-    jsonenc_putsep(e, ", ", &first);
-    jsonenc_value(e, elem.msg_val, values_m);
+      jsonenc_putsep(e, ",", &first);
+      jsonenc_value(e, elem.msg_val, values_m);
+    }
  }

  jsonenc_putstr(e, "]");
@ -564,7 +568,7 @@ static void jsonenc_mapkey(jsonenc *e, upb_msgval val, const upb_fielddef *f) {
      UPB_UNREACHABLE();
  }

-  jsonenc_putstr(e, "\": ");
+  jsonenc_putstr(e, "\":");
 }

 static void jsonenc_array(jsonenc *e, const upb_array *arr,
@ -576,7 +580,7 @@ static void jsonenc_array(jsonenc *e, const upb_array *arr,
  jsonenc_putstr(e, "[");

  for (i = 0; i < size; i++) {
-    jsonenc_putsep(e, ", ", &first);
+    jsonenc_putsep(e, ",", &first);
    jsonenc_scalar(e, upb_array_get(arr, i), f);
  }

@ -593,7 +597,7 @@ static void jsonenc_map(jsonenc *e, const upb_map *map, const upb_fielddef *f) {
  jsonenc_putstr(e, "{");

  while (upb_mapiter_next(map, &iter)) {
-    jsonenc_putsep(e, ", ", &first);
+    jsonenc_putsep(e, ",", &first);
    jsonenc_mapkey(e, upb_mapiter_key(map, iter), key_f);
    jsonenc_scalar(e, upb_mapiter_value(map, iter), val_f);
  }
@ -611,8 +615,8 @@ static void jsonenc_fieldval(jsonenc *e, const upb_fielddef *f,
    name = upb_fielddef_jsonname(f);
  }

-  jsonenc_putsep(e, ", ", first);
-  jsonenc_printf(e, "\"%s\": ", name);
+  jsonenc_putsep(e, ",", first);
+  jsonenc_printf(e, "\"%s\":", name);

  if (upb_fielddef_ismap(f)) {
    jsonenc_map(e, val.map_val, f);
@ -679,7 +683,7 @@ size_t upb_json_encode(const upb_msg *msg, const upb_msgdef *m,

  if (setjmp(e.err)) return -1;

-  jsonenc_msg(&e, msg, m);
+  jsonenc_msgfield(&e, msg, m);
  if (e.arena) upb_arena_free(e.arena);
  return jsonenc_nullz(&e, size);
 }
--- a/upb/msg.c
+++ b/upb/msg.c
@ -43,16 +43,14 @@ static const upb_msg_internal *upb_msg_getinternal_const(const upb_msg *msg) {
  return UPB_PTR_AT(msg, -sizeof(upb_msg_internal), upb_msg_internal);
 }

-static upb_msg_internal_withext *upb_msg_getinternalwithext(
-    upb_msg *msg, const upb_msglayout *l) {
-  UPB_ASSERT(l->extendable);
-  return UPB_PTR_AT(msg, -sizeof(upb_msg_internal_withext),
-                    upb_msg_internal_withext);
+void _upb_msg_clear(upb_msg *msg, const upb_msglayout *l) {
+  size_t internal = upb_msg_internalsize(l);
+  void *mem = UPB_PTR_AT(msg, -internal, char);
+  memset(mem, 0, l->size + internal);
 }

 upb_msg *_upb_msg_new(const upb_msglayout *l, upb_arena *a) {
  void *mem = upb_arena_malloc(a, upb_msg_sizeof(l));
-  upb_msg_internal *in;
  upb_msg *msg;

  if (!mem) {
@ -60,20 +58,7 @@ upb_msg *_upb_msg_new(const upb_msglayout *l, upb_arena *a) {
  }

  msg = UPB_PTR_AT(mem, upb_msg_internalsize(l), upb_msg);
-
-  /* Initialize normal members. */
-  memset(msg, 0, l->size);
-
-  /* Initialize internal members. */
-  in = upb_msg_getinternal(msg);
-  in->unknown = NULL;
-  in->unknown_len = 0;
-  in->unknown_size = 0;
-
-  if (l->extendable) {
-    upb_msg_getinternalwithext(msg, l)->extdict = NULL;
-  }
-
+  _upb_msg_clear(msg, l);
  return msg;
 }

@ -94,6 +79,11 @@ bool _upb_msg_addunknown(upb_msg *msg, const char *data, size_t len,
  return true;
 }

+void _upb_msg_discardunknown_shallow(upb_msg *msg) {
+  upb_msg_internal *in = upb_msg_getinternal(msg);
+  in->unknown_len = 0;
+}
+
 const char *upb_msg_getunknown(const upb_msg *msg, size_t *len) {
  const upb_msg_internal *in = upb_msg_getinternal_const(msg);
  *len = in->unknown_len;
--- a/upb/msg.h
+++ b/upb/msg.h
@ -81,6 +81,12 @@ extern char _upb_fieldtype_to_size[12];
 /* Creates a new messages with the given layout on the given arena. */
 upb_msg *_upb_msg_new(const upb_msglayout *l, upb_arena *a);

+/* Clears the given message. */
+void _upb_msg_clear(upb_msg *msg, const upb_msglayout *l);
+
+/* Discards the unknown fields for this message only. */
+void _upb_msg_discardunknown_shallow(upb_msg *msg);
+
 /* Adds unknown data (serialized protobuf data) to the given message.  The data
 * is copied into the message instance. */
 bool _upb_msg_addunknown(upb_msg *msg, const char *data, size_t len,
--- a/upb/reflection.c
+++ b/upb/reflection.c
@ -85,16 +85,22 @@ bool upb_msg_has(const upb_msg *msg, const upb_fielddef *f) {
  }
 }

-bool upb_msg_hasoneof(const upb_msg *msg, const upb_oneofdef *o) {
+const upb_fielddef *upb_msg_whichoneof(const upb_msg *msg,
+                                       const upb_oneofdef *o) {
  upb_oneof_iter i;
  const upb_fielddef *f;
  const upb_msglayout_field *field;
+  const upb_msgdef *m = upb_oneofdef_containingtype(o);
+  uint32_t oneof_case;

+  /* This is far from optimal. */
  upb_oneof_begin(&i, o);
  if (upb_oneof_done(&i)) return false;
  f = upb_oneof_iter_field(&i);
  field = upb_fielddef_layout(f);
-  return *oneofcase(msg, field) != 0;
+  oneof_case = *oneofcase(msg, field);
+
+  return oneof_case ? upb_msgdef_itof(m, oneof_case) : NULL;
 }

 upb_msgval upb_msg_get(const upb_msg *msg, const upb_fielddef *f) {
@ -164,6 +170,9 @@ upb_mutmsgval upb_msg_mutable(upb_msg *msg, const upb_fielddef *f,

    if (wrong_oneof) {
      *oneofcase(msg, field) = field->number;
+    } else if (field->presence > 0) {
+      uint32_t hasbit = field->presence;
+      *UPB_PTR_AT(msg, hasbit / 8, uint8_t) |= (1 << (hasbit % 8));
    }
  }
  return ret;
@ -176,11 +185,36 @@ void upb_msg_set(upb_msg *msg, const upb_fielddef *f, upb_msgval val,
  int size = upb_fielddef_isseq(f) ? sizeof(void *)
                                   : field_size[field->descriptortype];
  memcpy(mem, &val, size);
-  if (in_oneof(field)) {
+  if (field->presence > 0) {
+    uint32_t hasbit = field->presence;
+    *UPB_PTR_AT(msg, hasbit / 8, uint8_t) |= (1 << (hasbit % 8));
+  } else if (in_oneof(field)) {
    *oneofcase(msg, field) = field->number;
  }
 }

+void upb_msg_clearfield(upb_msg *msg, const upb_fielddef *f) {
+  const upb_msglayout_field *field = upb_fielddef_layout(f);
+  char *mem = UPB_PTR_AT(msg, field->offset, char);
+  int size = upb_fielddef_isseq(f) ? sizeof(void *)
+                                   : field_size[field->descriptortype];
+
+  if (field->presence > 0) {
+    uint32_t hasbit = field->presence;
+    *UPB_PTR_AT(msg, hasbit / 8, uint8_t) &= ~(1 << (hasbit % 8));
+  } else if (in_oneof(field)) {
+    uint32_t *oneof_case = oneofcase(msg, field);
+    if (*oneof_case != field->number) return;
+    *oneof_case = 0;
+  }
+
+  memset(mem, 0, size);
+}
+
+void upb_msg_clear(upb_msg *msg, const upb_msgdef *m) {
+  _upb_msg_clear(msg, upb_msgdef_layout(m));
+}
+
 bool upb_msg_next(const upb_msg *msg, const upb_msgdef *m,
                  const upb_symtab *ext_pool, const upb_fielddef **out_f,
                  upb_msgval *out_val, size_t *iter) {
@ -219,6 +253,56 @@ bool upb_msg_next(const upb_msg *msg, const upb_msgdef *m,
  return false;
 }

+bool _upb_msg_discardunknown(upb_msg *msg, const upb_msgdef *m, int depth) {
+  size_t iter = UPB_MSG_BEGIN;
+  const upb_fielddef *f;
+  upb_msgval val;
+  bool ret = true;
+
+  if (--depth == 0) return false;
+
+  _upb_msg_discardunknown_shallow(msg);
+
+  while (upb_msg_next(msg, m, NULL /*ext_pool*/, &f, &val, &iter)) {
+    const upb_msgdef *subm = upb_fielddef_msgsubdef(f);
+    if (!subm) continue;
+    if (upb_fielddef_ismap(f)) {
+      const upb_fielddef *val_f = upb_msgdef_itof(subm, 2);
+      const upb_msgdef *val_m = upb_fielddef_msgsubdef(val_f);
+      upb_map *map = (upb_map*)val.map_val;
+      size_t iter = UPB_MAP_BEGIN;
+
+      if (!val_m) continue;
+
+      while (upb_mapiter_next(map, &iter)) {
+        upb_msgval map_val = upb_mapiter_value(map, iter);
+        if (!_upb_msg_discardunknown((upb_msg*)map_val.msg_val, val_m, depth)) {
+          ret = false;
+        }
+      }
+    } else if (upb_fielddef_isseq(f)) {
+      const upb_array *arr = val.array_val;
+      size_t i, n = upb_array_size(arr);
+      for (i = 0; i < n; i++) {
+        upb_msgval elem = upb_array_get(arr, i);
+        if (!_upb_msg_discardunknown((upb_msg*)elem.msg_val, subm, depth)) {
+          ret = false;
+        }
+      }
+    } else {
+      if (!_upb_msg_discardunknown((upb_msg*)val.msg_val, subm, depth)) {
+        ret = false;
+      }
+    }
+  }
+
+  return ret;
+}
+
+bool upb_msg_discardunknown(upb_msg *msg, const upb_msgdef *m, int maxdepth) {
+  return _upb_msg_discardunknown(msg, m, maxdepth);
+}
+
 /** upb_array *****************************************************************/

 upb_array *upb_array_new(upb_arena *a, upb_fieldtype_t type) {
--- a/upb/reflection.h
+++ b/upb/reflection.h
@ -44,8 +44,9 @@ upb_mutmsgval upb_msg_mutable(upb_msg *msg, const upb_fielddef *f, upb_arena *a)
 /* May only be called for fields where upb_fielddef_haspresence(f) == true. */
 bool upb_msg_has(const upb_msg *msg, const upb_fielddef *f);

-/* Returns whether any field is set in the oneof. */
-bool upb_msg_hasoneof(const upb_msg *msg, const upb_oneofdef *o);
+/* Returns the field that is set in the oneof, or NULL if none are set. */
+const upb_fielddef *upb_msg_whichoneof(const upb_msg *msg,
+                                       const upb_oneofdef *o);

 /* Sets the given field to the given value.  For a msg/array/map/string, the
 * value must be in the same arena.  */
@ -55,6 +56,9 @@ void upb_msg_set(upb_msg *msg, const upb_fielddef *f, upb_msgval val,
 /* Clears any field presence and sets the value back to its default. */
 void upb_msg_clearfield(upb_msg *msg, const upb_fielddef *f);

+/* Clear all data and unknown fields. */
+void upb_msg_clear(upb_msg *msg, const upb_msgdef *m);
+
 /* Iterate over present fields.
 *
 * size_t iter = UPB_MSG_BEGIN;
@ -79,6 +83,9 @@ bool upb_msg_next(const upb_msg *msg, const upb_msgdef *m,
 void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len,
                        upb_arena *arena);

+/* Clears all unknown field data from this message and all submessages. */
+bool upb_msg_discardunknown(upb_msg *msg, const upb_msgdef *m, int maxdepth);
+
 /* Returns a reference to the message's unknown data. */
 const char *upb_msg_getunknown(const upb_msg *msg, size_t *len);

--- a/upbc/generator.cc
+++ b/upbc/generator.cc
@ -858,6 +858,7 @@ void WriteDefSource(const protobuf::FileDescriptor* file, Output& output) {
  EmitFileWarning(file, output);

  output("#include \"upb/def.h\"\n");
+  output("#include \"$0\"\n", DefHeaderFilename(file->name()));
  output("\n");

  for (int i = 0; i < file->dependency_count(); i++) {
@ -886,35 +887,19 @@ void WriteDefSource(const protobuf::FileDescriptor* file, Output& output) {
  std::string file_data;
  file_proto.SerializeToString(&file_data);

-  output("static const char descriptor[$0] =", file_data.size());
-
-  {
-    if (file_data.size() > 65535) {
-      // Workaround for MSVC: "Error C1091: compiler limit: string exceeds
-      // 65535 bytes in length". Declare a static array of chars rather than
-      // use a string literal. Only write 25 bytes per line.
-      static const size_t kBytesPerLine = 25;
-      output("{ ");
-      for (size_t i = 0; i < file_data.size();) {
-        for (size_t j = 0; j < kBytesPerLine && i < file_data.size(); ++i, ++j) {
-          output("'$0', ", absl::CEscape(file_data.substr(i, 1)));
-        }
-        output("\n");
-      }
-      output("'\\0' }");  // null-terminate
-    } else {
-      // Only write 40 bytes per line.
-      static const size_t kBytesPerLine = 40;
-      for (size_t i = 0; i < file_data.size(); i += kBytesPerLine) {
-        output("\n");
-        output(
-            "  \"$0\"",
-            EscapeTrigraphs(absl::CEscape(file_data.substr(i, kBytesPerLine))));
-      }
+  output("static const char descriptor[$0] = {", file_data.size());
+
+  // C90 only guarantees that strings can be up to 509 characters, and some
+  // implementations have limits here (for example, MSVC only allows 64k:
+  // https://docs.microsoft.com/en-us/cpp/error-messages/compiler-errors-1/fatal-error-c1091.
+  // So we always emit an array instead of a string.
+  for (size_t i = 0; i < file_data.size();) {
+    for (size_t j = 0; j < 25 && i < file_data.size(); ++i, ++j) {
+      output("'$0', ", absl::CEscape(file_data.substr(i, 1)));
    }
-    output(";\n");
+    output("\n");
  }
-  output("\n");
+  output("};\n\n");

  output("static upb_def_init *deps[$0] = {\n", file->dependency_count() + 1);
  for (int i = 0; i < file->dependency_count(); i++) {