Added comments.

python/BUILD

@@ -35,6 +35,8 @@ load(
 cc_binary(
     name = "message",
     srcs = [
+        "convert.c",
+        "convert.h",
         "descriptor.c",
         "descriptor.h",
         "descriptor_containers.c",

python/convert.c

@@ -0,0 +1,327 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/convert.h"
+
+#include "python/protobuf.h"
+#include "upb/reflection.h"
+
+PyObject* PyUpb_UpbToPy(upb_msgval val, const upb_fielddef *f, PyObject *arena) {
+  switch (upb_fielddef_type(f)) {
+    case UPB_TYPE_ENUM:
+    case UPB_TYPE_INT32:
+      return PyLong_FromLong(val.int32_val);
+    case UPB_TYPE_INT64:
+      return PyLong_FromLongLong(val.int64_val);
+    case UPB_TYPE_UINT32:
+      return PyLong_FromSize_t(val.uint32_val);
+    case UPB_TYPE_UINT64:
+      return PyLong_FromUnsignedLongLong(val.uint64_val);
+    case UPB_TYPE_FLOAT:
+      return PyFloat_FromDouble(val.float_val);
+    case UPB_TYPE_DOUBLE:
+      return PyFloat_FromDouble(val.double_val);
+    case UPB_TYPE_BOOL:
+      return PyBool_FromLong(val.bool_val);
+    case UPB_TYPE_BYTES:
+      return PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size);
+    case UPB_TYPE_STRING:
+      return PyUnicode_DecodeUTF8(val.str_val.data, val.str_val.size, NULL);
+    case UPB_TYPE_MESSAGE:
+      PyErr_Format(PyExc_NotImplementedError,
+                   "Conversion of message types not yet implemented");
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Getting a value from a field of unknown type %d",
+                   upb_fielddef_type(f));
+      return NULL;
+  }
+}
+
+static bool PyUpb_GetInt64(PyObject *obj, int64_t *val) {
+  // If the value is already a Python long, retrieves it.  Otherwise performs
+  // any automatic conversions to long (using __index__() or __int__()) that
+  // users expect.
+  *val = PyLong_AsLongLong(obj);
+  if (!PyErr_Occurred()) return true;
+  if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+    // Rewrite OverflowError -> ValueError.
+    // But don't rewrite other errors such as TypeError.
+    PyErr_Clear();
+    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+  }
+  return false;
+}
+
+static bool PyUpb_GetUint64(PyObject *obj, uint64_t *val) {
+  // For uint64 Python does not offer any functions as convenient as
+  // PyLong_AsLongLong().  If the object is not already a "long" we must
+  // manually perform the automatic conversion (using __index__() or __int__())
+  // that users expect.
+  if (PyLong_Check(obj)) {
+    *val = PyLong_AsUnsignedLongLong(obj);
+  } else {
+    PyObject* casted = PyNumber_Long(obj);
+    if (!casted) return false;
+    *val = PyLong_AsUnsignedLongLong(casted);
+    Py_DECREF(casted);
+  }
+  if (!PyErr_Occurred()) return true;
+  PyErr_Clear();
+  PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+  return false;
+}
+
+// If `arena` is specified, copies the string data into the given arena.
+// Otherwise aliases the given data.
+static upb_msgval PyUpb_MaybeCopyString(const char *ptr, size_t size,
+                                        upb_arena *arena) {
+  upb_msgval ret;
+  ret.str_val.size = size;
+  if (arena) {
+    char *buf = upb_arena_malloc(arena, size);
+    memcpy(buf, ptr, size);
+    ret.str_val.data = buf;
+  } else {
+    ret.str_val.data = ptr;
+  }
+  return ret;
+}
+
+#define CHECK_RANGE(in, out, low, high) \
+  ((out) = (in), ((out) >= (low) && (out) <= (high)))
+
+bool PyUpb_PyToUpb(PyObject *obj, const upb_fielddef *f, upb_msgval *val,
+                   upb_arena *arena) {
+  switch (upb_fielddef_type(f)) {
+    case UPB_TYPE_ENUM:
+      if (PyUnicode_Check(obj)) {
+        Py_ssize_t size;
+        const char *name = PyUnicode_AsUTF8AndSize(obj, &size);
+        const upb_enumdef *e = upb_fielddef_enumsubdef(f);
+        const upb_enumvaldef *ev = upb_enumdef_lookupname(e, name, size);
+        if (!ev) {
+          PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", name);
+          return false;
+        }
+        val->int32_val = upb_enumvaldef_number(ev);
+        return true;
+      } else {
+        if (!PyUpb_GetInt64(obj, &val->int64_val) ||
+            !CHECK_RANGE(val->int64_val, val->int32_val, INT32_MIN,
+                         INT32_MAX)) {
+          return false;
+        }
+        const upb_enumdef *e = upb_fielddef_enumsubdef(f);
+        if (upb_filedef_syntax(upb_enumdef_file(e)) == UPB_SYNTAX_PROTO2 &&
+            !upb_enumdef_checknum(e, val->int32_val)) {
+          PyErr_Format(PyExc_ValueError, "invalid enumerator %d",
+                       (int)val->int32_val);
+          return false;
+        }
+        return true;
+      }
+    case UPB_TYPE_INT32:
+      return PyUpb_GetInt64(obj, &val->int64_val) &&
+             CHECK_RANGE(val->int64_val, val->int32_val, INT32_MIN, INT32_MAX);
+    case UPB_TYPE_INT64:
+      return PyUpb_GetInt64(obj, &val->int64_val);
+    case UPB_TYPE_UINT32:
+      return PyUpb_GetUint64(obj, &val->uint64_val) &&
+             CHECK_RANGE(val->uint64_val, val->uint32_val, 0, UINT32_MAX);
+    case UPB_TYPE_UINT64:
+      return PyUpb_GetUint64(obj, &val->uint64_val);
+    case UPB_TYPE_FLOAT:
+      val->float_val = PyFloat_AsDouble(obj);
+      return !PyErr_Occurred();
+    case UPB_TYPE_DOUBLE:
+      val->double_val = PyFloat_AsDouble(obj);
+      return !PyErr_Occurred();
+    case UPB_TYPE_BOOL:
+      val->bool_val = PyLong_AsLong(obj);
+      return !PyErr_Occurred();
+    case UPB_TYPE_BYTES: {
+      char *ptr;
+      Py_ssize_t size;
+      if (PyBytes_AsStringAndSize(obj, &ptr, &size) < 0) return false;
+      *val = PyUpb_MaybeCopyString(ptr, size, arena);
+      return true;
+    }
+    case UPB_TYPE_STRING: {
+      Py_ssize_t size;
+      const char *ptr;
+      PyObject *unicode = NULL;
+      if (PyBytes_Check(obj)) {
+        unicode = obj = PyUnicode_FromEncodedObject(obj, "utf-8", NULL);
+        if (!obj) return false;
+      }
+      ptr = PyUnicode_AsUTF8AndSize(obj, &size);
+      if (PyErr_Occurred()) {
+        Py_XDECREF(unicode);
+        return false;
+      }
+      *val = PyUpb_MaybeCopyString(ptr, size, arena);
+      Py_XDECREF(unicode);
+      return true;
+    }
+    case UPB_TYPE_MESSAGE:
+      PyErr_Format(
+          PyExc_ValueError, "Message objects may not be assigned",
+          upb_fielddef_type(f));
+      return false;
+    default:
+      PyErr_Format(
+          PyExc_SystemError, "Getting a value from a field of unknown type %d",
+          upb_fielddef_type(f));
+      return false;
+  }
+}
+
+bool PyUpb_Message_IsEqual(const upb_msg *msg1, const upb_msg *msg2,
+                           const upb_msgdef *m);
+
+// -----------------------------------------------------------------------------
+// Equal
+// -----------------------------------------------------------------------------
+
+bool PyUpb_ValueEq(upb_msgval val1, upb_msgval val2, const upb_fielddef *f) {
+  switch (upb_fielddef_type(f)) {
+    case UPB_TYPE_BOOL:
+      return val1.bool_val == val2.bool_val;
+    case UPB_TYPE_INT32:
+    case UPB_TYPE_UINT32:
+    case UPB_TYPE_ENUM:
+      return val1.int32_val == val2.int32_val;
+    case UPB_TYPE_INT64:
+    case UPB_TYPE_UINT64:
+      return val1.int64_val == val2.int64_val;
+    case UPB_TYPE_FLOAT:
+      return val1.float_val == val2.float_val;
+    case UPB_TYPE_DOUBLE:
+      return val1.double_val == val2.double_val;
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES:
+      return val1.str_val.size == val2.str_val.size &&
+          memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == 0;
+    case UPB_TYPE_MESSAGE:
+      return PyUpb_Message_IsEqual(val1.msg_val, val2.msg_val,
+                                   upb_fielddef_msgsubdef(f));
+    default:
+      return false;
+  }
+}
+
+bool PyUpb_Map_IsEqual(const upb_map *map1, const upb_map *map2,
+                       const upb_fielddef *f) {
+  assert(upb_fielddef_ismap(f));
+  const upb_msgdef *entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef *val_f = upb_msgdef_field(entry_m, 1);
+  size_t iter = UPB_MAP_BEGIN;
+  size_t size1 = map1 ? upb_map_size(map1) : 0;
+  size_t size2 = map2 ? upb_map_size(map2) : 0;
+
+  if (map1 == map2) return true;
+  if (size1 != size2) return false;
+  if (size1 == 0) return true;
+
+  while (upb_mapiter_next(map1, &iter)) {
+    upb_msgval key = upb_mapiter_key(map1, iter);
+    upb_msgval val1 = upb_mapiter_value(map1, iter);
+    upb_msgval val2;
+    if (!upb_map_get(map2, key, &val2)) return false;
+    if (!PyUpb_ValueEq(val1, val2, val_f)) return false;
+  }
+
+  return true;
+}
+
+static bool PyUpb_ArrayElem_IsEqual(const upb_array *arr1,
+                                    const upb_array *arr2, size_t i,
+                                    const upb_fielddef *f) {
+  upb_msgval val1 = upb_array_get(arr1, i);
+  upb_msgval val2 = upb_array_get(arr2, i);
+  return PyUpb_ValueEq(val1, val2, f);
+}
+
+bool PyUpb_Array_IsEqual(const upb_array *arr1, const upb_array *arr2,
+                         const upb_fielddef *f) {
+  assert(upb_fielddef_isseq(f) && !upb_fielddef_ismap(f));
+  size_t n1 = arr1 ? upb_array_size(arr1) : 0;
+  size_t n2 = arr2 ? upb_array_size(arr2) : 0;
+
+  if (arr1 == arr2) return true;
+  if (n1 != n2) return false;
+
+  // Half the length rounded up.  Important: the empty list rounds to 0.
+  size_t half = (n1 + 1) / 2;
+
+  // Search from the ends-in.  We expect differences to more quickly manifest
+  // at the ends than in the middle.  We may compare the most middle element
+  // twice.
+  for (size_t i = 0; i < half; i++) {
+    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, i, f)) return false;
+    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, n1 - 1 - i, f)) return false;
+  }
+
+  return true;
+}
+
+bool PyUpb_Message_IsEqual(const upb_msg *msg1, const upb_msg *msg2,
+                           const upb_msgdef *m) {
+  size_t iter1 = UPB_MSG_BEGIN;
+  size_t iter2 = UPB_MSG_BEGIN;
+
+  if (msg1 == msg2) return true;
+
+  while (true) {
+    const upb_fielddef *f1, *f2;
+    upb_msgval val1, val2;
+    bool ok1 = msg1 && upb_msg_next(msg1, m, NULL, &f1, &val1, &iter1);
+    bool ok2 = msg2 && upb_msg_next(msg2, m, NULL, &f2, &val2, &iter2);
+    if (ok1 != ok2) return false;
+    if (!ok1) break;  // Both messages are at end.
+    if (f1 != f2) return false;
+    if (upb_fielddef_ismap(f1)) {
+      if (!PyUpb_Map_IsEqual(val1.map_val, val2.map_val, f1)) return false;
+    } else if (upb_fielddef_isseq(f1)) {
+      if (!PyUpb_Array_IsEqual(val1.array_val, val2.array_val, f1)) {
+        return false;
+      }
+    } else {
+      if (!PyUpb_ValueEq(val1, val2, f1)) return false;
+    }
+  }
+
+  size_t usize1, usize2;
+  const char *uf1 = upb_msg_getunknown(msg1, &usize1);
+  const char *uf2 = upb_msg_getunknown(msg2, &usize2);
+  // TODO(haberman): compare unknown fields one by one.
+  if (usize1 != usize2) return false;
+  if (memcmp(uf1, uf2, usize1)) return false;
+
+  return true;
+}

python/convert.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_CONVERT_H__
+#define PYUPB_CONVERT_H__
+
+#include "upb/def.h"
+#include "upb/reflection.h"
+
+#include "protobuf.h"
+
+// Converts `val` to a Python object according to the type information in `f`.
+// Any newly-created objects that reference non-primitive data from `val` should
+// reference `arena` to keep the referent alive.  If the conversion cannot be
+// performed, returns NULL and sets a Python error.
+PyObject *PyUpb_UpbToPy(upb_msgval val, const upb_fielddef *f, PyObject *arena);
+
+// Converts `obj` to a upb_msgval `*val` according to the type information in
+// `f`. If `arena is provided, any string data will be copied into `arena`,
+// otherwise the returned value will alias the Python-owned data (this can be
+// useful for an ephemeral upb_msgval).  If the conversion cannot be performed,
+// returns false.
+bool PyUpb_PyToUpb(PyObject *obj, const upb_fielddef *f, upb_msgval *val,
+                   upb_arena *arena);
+
+// Returns true if the given values (of type `f`) are equal.
+bool PyUpb_ValueEq(upb_msgval val1, upb_msgval val2, const upb_fielddef *f);
+
+// Returns true if the given messages (of type `m`) are equal.
+bool PyUpb_Message_IsEqual(const upb_msg *msg1, const upb_msg *msg2,
+                           const upb_msgdef *m);
+
+// Returns true if the two arrays (with element type `f`) are equal.
+bool PyUpb_Array_IsEqual(const upb_array *arr1, const upb_array *arr2,
+                         const upb_fielddef *f);
+
+#endif  // PYUPB_CONVERT_H__

upb/def.c

@@ -394,6 +394,12 @@ const upb_enumvaldef *upb_enumdef_lookupnum(const upb_enumdef *def, int32_t num)
                                                   : NULL;
 }
 
+bool upb_enumdef_checknum(const upb_enumdef *e, int32_t num) {
+  // We could use upb_enumdef_lookupnum(e, num) != NULL, but we expect this to
+  // be faster (especially for small numbers).
+  return _upb_enumlayout_checkval(e->layout, num);
+}
+
 const upb_enumvaldef *upb_enumdef_value(const upb_enumdef *e, int i) {
   UPB_ASSERT(0 <= i && i < e->value_count);
   return &e->values[i];

upb/def.h

@@ -318,6 +318,7 @@ const upb_enumvaldef *upb_enumdef_value(const upb_enumdef *e, int i);
 const upb_enumvaldef *upb_enumdef_lookupname(const upb_enumdef *e,
                                              const char *name, size_t len);
 const upb_enumvaldef *upb_enumdef_lookupnum(const upb_enumdef *e, int32_t num);
+bool upb_enumdef_checknum(const upb_enumdef *e, int32_t num);
 
 /* DEPRECATED, slated for removal */
 int upb_enumdef_numvals(const upb_enumdef *e);