Functionality is complete and it is passing all tests.

upb/util/BUILD

@@ -32,7 +32,9 @@ cc_test(
     name = "required_fields_test",
     srcs = ["required_fields_test.cc"],
     deps = [
+        "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest_main",
+        "//:json",
         ":required_fields",
         ":required_fields_test_upb_proto",
         ":required_fields_test_upb_proto_reflection",

upb/util/README.md

@@ -0,0 +1,7 @@
+
+# upb util library
+
+The libraries in this directory contain useful functionality that is layered
+on top of the main upb APIs.  In other words, the APIs in this directory have
+no special access to upb internals; you could easily implement the same things
+yourself.

upb/util/required_fields.c

@@ -27,6 +27,7 @@
 
 #include "upb/util/required_fields.h"
 
+#include <inttypes.h>
 #include <setjmp.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -36,6 +37,117 @@
 // Must be last.
 #include "upb/port_def.inc"
 
+////////////////////////////////////////////////////////////////////////////////
+// upb_FieldPath_ToText()
+////////////////////////////////////////////////////////////////////////////////
+
+typedef struct {
+  char* buf;
+  char* ptr;
+  char* end;
+  size_t overflow;
+} upb_PrintfAppender;
+
+UPB_PRINTF(2, 3)
+static void upb_FieldPath_Printf(upb_PrintfAppender* a, const char* fmt, ...) {
+  size_t n;
+  size_t have = a->end - a->ptr;
+  va_list args;
+
+  va_start(args, fmt);
+  n = vsnprintf(a->ptr, have, fmt, args);
+  va_end(args);
+
+  if (UPB_LIKELY(have > n)) {
+    a->ptr += n;
+  } else {
+    a->ptr = UPB_PTRADD(a->ptr, have);
+    a->overflow += (n - have);
+  }
+}
+
+static size_t upb_FieldPath_NullTerminate(upb_PrintfAppender *d, size_t size) {
+  size_t ret = d->ptr - d->buf + d->overflow;
+
+  if (size > 0) {
+    if (d->ptr == d->end) d->ptr--;
+    *d->ptr = '\0';
+  }
+
+  return ret;
+}
+
+static void upb_FieldPath_PutMapKey(upb_PrintfAppender *a, upb_msgval map_key,
+                                    const upb_fielddef *key_f) {
+  switch (upb_fielddef_type(key_f)) {
+    case UPB_TYPE_INT32:
+      upb_FieldPath_Printf(a, "[%" PRId32 "]", map_key.int32_val);
+      break;
+    case UPB_TYPE_INT64:
+      upb_FieldPath_Printf(a, "[%" PRId64 "]", map_key.int64_val);
+      break;
+    case UPB_TYPE_UINT32:
+      upb_FieldPath_Printf(a, "[%" PRIu32 "]", map_key.uint32_val);
+      break;
+    case UPB_TYPE_UINT64:
+      upb_FieldPath_Printf(a, "[%" PRIu64 "]", map_key.uint64_val);
+      break;
+    case UPB_TYPE_BOOL:
+      upb_FieldPath_Printf(a, "[%s]", map_key.bool_val ? "true" : "false");
+      break;
+    case UPB_TYPE_STRING:
+      upb_FieldPath_Printf(a, "[\"");
+      for (size_t i = 0; i < map_key.str_val.size; i++) {
+        char ch = map_key.str_val.data[i];
+        if (ch == '"') {
+          upb_FieldPath_Printf(a, "\\\"");
+        } else {
+          upb_FieldPath_Printf(a, "%c", ch);
+        }
+      }
+      upb_FieldPath_Printf(a, "\"]");
+      break;
+    default:
+      UPB_UNREACHABLE();  // Other types can't be map keys.
+  }
+}
+
+size_t upb_FieldPath_ToText(upb_FieldPathEntry** path, char* buf, size_t size) {
+  upb_FieldPathEntry *ptr = *path;
+  upb_PrintfAppender appender;
+  appender.buf = buf;
+  appender.ptr = buf;
+  appender.end = UPB_PTRADD(buf, size);
+  appender.overflow = 0;
+  bool first = true;
+
+  while (ptr->field) {
+    const upb_fielddef *f = ptr->field;
+
+    upb_FieldPath_Printf(&appender, first ? "%s" : ".%s", upb_fielddef_name(f));
+    first = false;
+    ptr++;
+
+    if (upb_fielddef_ismap(f)) {
+      const upb_fielddef *key_f = upb_msgdef_field(upb_fielddef_msgsubdef(f), 0);
+      upb_FieldPath_PutMapKey(&appender, ptr->map_key, key_f);
+      ptr++;
+    } else if (upb_fielddef_isseq(f)) {
+      upb_FieldPath_Printf(&appender, "[%zu]", ptr->array_index);
+      ptr++;
+    }
+  }
+
+  // Advance beyond terminating NULL.
+  ptr++;
+  *path = ptr;
+  return upb_FieldPath_NullTerminate(&appender, size);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// upb_util_HasUnsetRequired()
+////////////////////////////////////////////////////////////////////////////////
+
 typedef struct {
   upb_FieldPathEntry *path;
   size_t size;
@@ -51,14 +163,15 @@ typedef struct {
   bool save_paths;
 } upb_FindContext;
 
-void upb_FieldPathVector_Init(upb_FieldPathVector *vec) {
+static void upb_FieldPathVector_Init(upb_FieldPathVector *vec) {
   vec->path = NULL;
   vec->size = 0;
   vec->cap = 0;
 }
 
-void upb_FieldPathVector_Reserve(upb_FindContext *ctx, upb_FieldPathVector *vec,
-                                 size_t elems) {
+static void upb_FieldPathVector_Reserve(upb_FindContext *ctx,
+                                        upb_FieldPathVector *vec,
+                                        size_t elems) {
   if (vec->cap - vec->size < elems) {
     size_t need = vec->size + elems;
     vec->cap = UPB_MAX(4, vec->cap);
@@ -68,41 +181,46 @@ void upb_FieldPathVector_Reserve(upb_FindContext *ctx, upb_FieldPathVector *vec,
   }
 }
 
-void upb_FindContext_Push(upb_FindContext* ctx, upb_FieldPathEntry ent) {
+static void upb_FindContext_Push(upb_FindContext* ctx, upb_FieldPathEntry ent) {
   if (!ctx->save_paths) return;
   upb_FieldPathVector_Reserve(ctx, &ctx->stack, 1);
   ctx->stack.path[ctx->stack.size++] = ent;
 }
 
-void upb_FindContext_Pop(upb_FindContext* ctx) {
+static void upb_FindContext_Pop(upb_FindContext* ctx) {
   if (!ctx->save_paths) return;
   assert(ctx->stack.size != 0);
   ctx->stack.size--;
 }
 
-static void upb_util_FindUnsetRequiredInternal(upb_FindContext *ctx,
-                                               const upb_msg *msg,
-                                               const upb_msgdef *m) {
+static void upb_util_FindUnsetRequiredInternal(upb_FindContext* ctx,
+                                               const upb_msg* msg,
+                                               const upb_msgdef* m) {
   // OPT: add markers in the schema for where we can avoid iterating:
   // 1. messages with no required fields.
   // 2. messages that cannot possibly reach any required fields.
 
   // Iterate over all fields to see if any required fields are missing.
   for (int i = 0, n = upb_msgdef_fieldcount(m); i < n; i++) {
-    const upb_fielddef *f = upb_msgdef_field(m, i);
+    const upb_fielddef* f = upb_msgdef_field(m, i);
     if (upb_fielddef_label(f) != UPB_LABEL_REQUIRED) continue;
 
     if (!upb_msg_has(msg, f)) {
       // A required field is missing.
       ctx->has_unset_required = true;
 
-      // Append the contents of the stack to the out array, then NULL-terminate.
-      upb_FieldPathVector_Reserve(ctx, &ctx->out_fields, ctx->stack.size + 1);
-      memcpy(&ctx->out_fields.path[ctx->out_fields.size], &ctx->stack.path,
-             ctx->stack.size * sizeof(*ctx->stack.path));
-      ctx->out_fields.size += ctx->stack.size;
-      ctx->out_fields.path[ctx->out_fields.size++] =
-          (upb_FieldPathEntry){.field = NULL};
+      if (ctx->save_paths) {
+        // Append the contents of the stack to the out array, then
+        // NULL-terminate.
+        upb_FieldPathVector_Reserve(ctx, &ctx->out_fields, ctx->stack.size + 2);
+        memcpy(&ctx->out_fields.path[ctx->out_fields.size], ctx->stack.path,
+               ctx->stack.size * sizeof(*ctx->stack.path));
+        ctx->out_fields.size += ctx->stack.size;
+        ctx->out_fields.path[ctx->out_fields.size++] =
+            (upb_FieldPathEntry){.field = f};
+        ctx->out_fields.path[ctx->out_fields.size++] =
+            (upb_FieldPathEntry){.field = NULL};
+      }
     }
   }
 
@@ -114,40 +232,40 @@ static void upb_util_FindUnsetRequiredInternal(upb_FindContext *ctx,
   // TODO(haberman): consider changing upb_msg_next() to be capable of visiting
   // extensions only, for example with a UPB_MSG_BEGINEXT constant.
   size_t iter = UPB_MSG_BEGIN;
-  const upb_fielddef *f;
+  const upb_fielddef* f;
   upb_msgval val;
   while (upb_msg_next(msg, m, ctx->ext_pool, &f, &val, &iter)) {
     // Skip non-submessage fields.
     if (!upb_fielddef_issubmsg(f)) continue;
 
     upb_FindContext_Push(ctx, (upb_FieldPathEntry){.field = f});
-    const upb_msgdef *sub_m = upb_fielddef_msgsubdef(f);
+    const upb_msgdef* sub_m = upb_fielddef_msgsubdef(f);
 
     if (upb_fielddef_ismap(f)) {
       // Map field.
-      const upb_fielddef *val_f = upb_msgdef_field(sub_m, 1);
-      const upb_msgdef *val_m = upb_fielddef_msgsubdef(val_f);
+      const upb_fielddef* val_f = upb_msgdef_field(sub_m, 1);
+      const upb_msgdef* val_m = upb_fielddef_msgsubdef(val_f);
       if (!val_m) continue;
-      const upb_map *map = val.map_val;
+      const upb_map* map = val.map_val;
       size_t iter = UPB_MAP_BEGIN;
       while (upb_mapiter_next(map, &iter)) {
-        upb_msgval key = upb_mapiter_value(map, iter);
+        upb_msgval key = upb_mapiter_key(map, iter);
         upb_msgval map_val = upb_mapiter_value(map, iter);
-        upb_FindContext_Push(ctx, (upb_FieldPathEntry){.key = key});
+        upb_FindContext_Push(ctx, (upb_FieldPathEntry){.map_key = key});
         upb_util_FindUnsetRequiredInternal(ctx, map_val.msg_val, val_m);
         upb_FindContext_Pop(ctx);
       }
     } else if (upb_fielddef_isseq(f)) {
       // Repeated field.
-      const upb_array *arr = val.array_val;
+      const upb_array* arr = val.array_val;
       for (size_t i = 0, n = upb_array_size(arr); i < n; i++) {
         upb_msgval elem = upb_array_get(arr, i);
-        upb_FindContext_Push(ctx, (upb_FieldPathEntry){.index = i});
+        upb_FindContext_Push(ctx, (upb_FieldPathEntry){.array_index = i});
         upb_util_FindUnsetRequiredInternal(ctx, elem.msg_val, sub_m);
         upb_FindContext_Pop(ctx);
       }
     } else {
-      // Scalar field.
+      // Scalar sub-message field.
       upb_util_FindUnsetRequiredInternal(ctx, val.msg_val, sub_m);
     }
 
@@ -159,11 +277,18 @@ bool upb_util_HasUnsetRequired(const upb_msg* msg, const upb_msgdef* m,
                                const upb_symtab* ext_pool,
                                upb_FieldPathEntry** fields) {
   upb_FindContext ctx;
+  ctx.has_unset_required = false;
+  ctx.save_paths = fields != NULL;
   ctx.ext_pool = ext_pool;
   upb_FieldPathVector_Init(&ctx.stack);
   upb_FieldPathVector_Init(&ctx.out_fields);
   upb_util_FindUnsetRequiredInternal(&ctx, msg, m);
   free(ctx.stack.path);
-  if (fields) *fields = ctx.out_fields.path;
+  if (fields) {
+    upb_FieldPathVector_Reserve(&ctx, &ctx.out_fields, 1);
+    ctx.out_fields.path[ctx.out_fields.size] =
+        (upb_FieldPathEntry){.field = NULL};
+    *fields = ctx.out_fields.path;
+  }
   return ctx.has_unset_required;
 }

upb/util/required_fields.h

@@ -48,10 +48,20 @@ extern "C" {
 // map key after it.
 typedef union {
   const upb_fielddef* field;
-  size_t index;
-  upb_msgval key;
+  size_t array_index;
+  upb_msgval map_key;
 } upb_FieldPathEntry;
 
+// Writes a string representing `*path` to `buf` in the following textual format:
+//    foo.bar.repeated_baz[2].string_msg_map["abc"]
+//
+// The given buffer will always be NULL-terminated. If the data (including NULL
+// terminator) exceeds `size`, the result will be truncated.
+//
+// The pointer `*path` will be updated to point to one past the terminating NULL
+// pointer of the input array.
+size_t upb_FieldPath_ToText(upb_FieldPathEntry **path, char *buf, size_t size);
+
 // Checks whether `msg` or any of its children has unset required fields,
 // returning `true` if any are found.
 //

upb/util/required_fields_test.cc

@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "upb/util/required_fields.h"
+
+#include "absl/strings/string_view.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "upb/def.hpp"
+#include "upb/json_decode.h"
+#include "upb/upb.hpp"
+#include "upb/util/required_fields_test.upb.h"
+#include "upb/util/required_fields_test.upbdefs.h"
+
+std::vector<std::string> PathsToText(upb_FieldPathEntry *entry) {
+  std::vector<std::string> ret;
+  char buf[1024];  // Larger than anything we'll use in this test.
+  while(entry->field) {
+    upb_FieldPathEntry *before = entry;
+    size_t len = upb_FieldPath_ToText(&entry, buf, sizeof(buf));
+    EXPECT_LT(len, sizeof(buf));
+    assert(len <= sizeof(buf));
+    ret.push_back(buf);
+
+    // Ensure that we can have a short buffer and that it will be
+    // NULL-terminated.
+    char shortbuf[4];
+    size_t len2 = upb_FieldPath_ToText(&before, shortbuf, sizeof(shortbuf));
+    EXPECT_EQ(len, len2);
+    EXPECT_EQ(ret.back().substr(0, sizeof(shortbuf) - 1),
+              std::string(shortbuf));
+  }
+  return ret;
+}
+
+void CheckRequired(absl::string_view json, const std::vector<std::string>& missing) {
+  upb::Arena arena;
+  upb::SymbolTable symtab;
+  upb_util_test_TestRequiredFields *test_msg =
+      upb_util_test_TestRequiredFields_new(arena.ptr());
+  upb::MessageDefPtr m(
+      upb_util_test_TestRequiredFields_getmsgdef(symtab.ptr()));
+  upb::Status status;
+  EXPECT_TRUE(upb_json_decode(json.data(), json.size(), test_msg, m.ptr(),
+                              symtab.ptr(), 0, arena.ptr(), status.ptr()))
+      << status.error_message();
+  upb_FieldPathEntry *entries;
+  EXPECT_EQ(!missing.empty(), upb_util_HasUnsetRequired(
+                                  test_msg, m.ptr(), symtab.ptr(), &entries));
+  EXPECT_EQ(missing, PathsToText(entries));
+  free(entries);
+
+  // Verify that we can pass a NULL pointer to entries when we don't care about them.
+  EXPECT_EQ(!missing.empty(),
+            upb_util_HasUnsetRequired(test_msg, m.ptr(), symtab.ptr(), NULL));
+}
+
+// message HasRequiredField {
+//   required int32 required_int32 = 1;
+// }
+// 
+// message TestRequiredFields {
+//   required EmptyMessage required_message = 1;
+//   optional TestRequiredFields optional_message = 2;
+//   repeated HasRequiredField repeated_message = 3;
+//   map<int32, HasRequiredField> map_int32_message = 4;
+// }
+TEST(RequiredFieldsTest, TestRequired) {
+  CheckRequired(R"json({})json", {"required_message"});
+  CheckRequired(R"json({"required_message": {}}")json", {});
+  CheckRequired(
+      R"json(
+      {
+        "optional_message": {}
+      }
+      )json",
+      {"required_message", "optional_message.required_message"});
+
+  // Repeated field.
+  CheckRequired(
+      R"json(
+      {
+        "optional_message": {
+          "repeated_message": [
+            {"required_int32": 1},
+            {},
+            {"required_int32": 2}
+          ]
+        }
+      }
+      )json",
+      {"required_message", "optional_message.required_message",
+       "optional_message.repeated_message[1].required_int32"});
+  
+  // Int32 map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_int32_message": {
+          "1": {"required_int32": 1},
+          "5": {},
+          "9": {"required_int32": 1}
+        }
+      }
+      )json",
+      {"map_int32_message[5].required_int32"});
+
+  // Int64 map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_int64_message": {
+          "1": {"required_int32": 1},
+          "5": {},
+          "9": {"required_int32": 1}
+        }
+      }
+      )json",
+      {"map_int64_message[5].required_int32"});
+
+  // Uint32 map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_uint32_message": {
+          "1": {"required_int32": 1},
+          "5": {},
+          "9": {"required_int32": 1}
+        }
+      }
+      )json",
+      {"map_uint32_message[5].required_int32"});
+
+  // Uint64 map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_uint64_message": {
+          "1": {"required_int32": 1},
+          "5": {},
+          "9": {"required_int32": 1}
+        }
+      }
+      )json",
+      {"map_uint64_message[5].required_int32"});
+
+  // Bool map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_bool_message": {
+          "false": {"required_int32": 1},
+          "true": {}
+        }
+      }
+      )json",
+      {"map_bool_message[true].required_int32"});
+
+  // String map key.
+  CheckRequired(
+      R"json(
+      {
+        "required_message": {},
+        "map_string_message": {
+          "abc": {"required_int32": 1},
+          "d\"ef": {}
+        }
+      }
+      )json",
+      {R"(map_string_message["d\"ef"].required_int32)"});
+}

upb/util/required_fields_test.proto

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+syntax = "proto2";
+
+package upb_util_test;
+
+message EmptyMessage {}
+
+message HasRequiredField {
+  required int32 required_int32 = 1;
+}
+
+message TestRequiredFields {
+  required EmptyMessage required_message = 1;
+  optional TestRequiredFields optional_message = 2;
+  repeated HasRequiredField repeated_message = 3;
+  map<int32, HasRequiredField> map_int32_message = 4;
+  map<int64, HasRequiredField> map_int64_message = 5;
+  map<uint32, HasRequiredField> map_uint32_message = 6;
+  map<uint64, HasRequiredField> map_uint64_message = 7;
+  map<bool, HasRequiredField> map_bool_message = 8;
+  map<string, HasRequiredField> map_string_message = 9;
+}