protobuf/python/convert.c

/*
 * Copyright (c) 2009-2021, Google LLC
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Google LLC nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "python/convert.h"

#include "python/message.h"
#include "python/protobuf.h"
#include "upb/reflection.h"
#include "upb/util/compare.h"

PyObject* PyUpb_UpbToPy(upb_msgval val, const upb_fielddef *f, PyObject *arena) {
  switch (upb_fielddef_type(f)) {
    case UPB_TYPE_ENUM:
    case UPB_TYPE_INT32:
      return PyLong_FromLong(val.int32_val);
    case UPB_TYPE_INT64:
      return PyLong_FromLongLong(val.int64_val);
    case UPB_TYPE_UINT32:
      return PyLong_FromSize_t(val.uint32_val);
    case UPB_TYPE_UINT64:
      return PyLong_FromUnsignedLongLong(val.uint64_val);
    case UPB_TYPE_FLOAT:
      return PyFloat_FromDouble(val.float_val);
    case UPB_TYPE_DOUBLE:
      return PyFloat_FromDouble(val.double_val);
    case UPB_TYPE_BOOL:
      return PyBool_FromLong(val.bool_val);
    case UPB_TYPE_BYTES:
      return PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size);
    case UPB_TYPE_STRING: {
      PyObject* ret =
          PyUnicode_DecodeUTF8(val.str_val.data, val.str_val.size, NULL);
      // If the string can't be decoded in UTF-8, just return a bytes object
      // that contains the raw bytes. This can't happen if the value was
      // assigned using the members of the Python message object, but can happen
      // if the values were parsed from the wire (binary).
      if (ret == NULL) {
        PyErr_Clear();
        ret = PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size);
      }
      return ret;
    }
    case UPB_TYPE_MESSAGE:
      return PyUpb_CMessage_Get((upb_msg*)val.msg_val,
                                upb_fielddef_msgsubdef(f), arena);
    default:
      PyErr_Format(PyExc_SystemError,
                   "Getting a value from a field of unknown type %d",
                   upb_fielddef_type(f));
      return NULL;
  }
}

static bool PyUpb_GetInt64(PyObject *obj, int64_t *val) {
  // We require that the value is either an integer or has an __index__
  // conversion.
  if (!PyIndex_Check(obj)) {
    PyErr_Format(PyExc_TypeError, "Expected integer: %S", obj);
  }
  // If the value is already a Python long, PyLong_AsLongLong() retrieves it.
  // Otherwise is converts to integer using __int__.
  *val = PyLong_AsLongLong(obj);
  if (!PyErr_Occurred()) return true;
  if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
    // Rewrite OverflowError -> ValueError.
    // But don't rewrite other errors such as TypeError.
    PyErr_Clear();
    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
  }
  return false;
}

static bool PyUpb_GetUint64(PyObject *obj, uint64_t *val) {
  // We require that the value is either an integer or has an __index__
  // conversion.
  if (!PyIndex_Check(obj)) {
    PyErr_Format(PyExc_TypeError, "Expected integer: %S", obj);
  }
  if (PyLong_Check(obj)) {
    *val = PyLong_AsUnsignedLongLong(obj);
  } else if (PyIndex_Check(obj)) {
    PyObject* casted = PyNumber_Long(obj);
    if (!casted) return false;
    *val = PyLong_AsUnsignedLongLong(casted);
    Py_DECREF(casted);
  } else {
    PyErr_Format(PyExc_TypeError, "Expected integer: %S", obj);
    return false;
  }
  if (!PyErr_Occurred()) return true;
  PyErr_Clear();
  PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
  return false;
}

static bool PyUpb_GetInt32(PyObject *obj, int32_t *val) {
  int64_t i64;
  if (!PyUpb_GetInt64(obj, &i64)) return false;
  if (i64 < INT32_MIN || i64 > INT32_MAX) {
    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
    return false;
  }
  *val = i64;
  return true;
}

static bool PyUpb_GetUint32(PyObject *obj, uint32_t *val) {
  uint64_t u64;
  if (!PyUpb_GetUint64(obj, &u64)) return false;
  if (u64 > UINT32_MAX) {
    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
    return false;
  }
  *val = u64;
  return true;
}

// If `arena` is specified, copies the string data into the given arena.
// Otherwise aliases the given data.
static upb_msgval PyUpb_MaybeCopyString(const char *ptr, size_t size,
                                        upb_arena *arena) {
  upb_msgval ret;
  ret.str_val.size = size;
  if (arena) {
    char *buf = upb_arena_malloc(arena, size);
    memcpy(buf, ptr, size);
    ret.str_val.data = buf;
  } else {
    ret.str_val.data = ptr;
  }
  return ret;
}

static bool PyUpb_PyToUpbEnum(PyObject *obj, const upb_enumdef *e,
                              upb_msgval *val) {
  if (PyUnicode_Check(obj)) {
    Py_ssize_t size;
    const char *name = PyUnicode_AsUTF8AndSize(obj, &size);
    const upb_enumvaldef *ev = upb_enumdef_lookupname(e, name, size);
    if (!ev) {
      PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", name);
      return false;
    }
    val->int32_val = upb_enumvaldef_number(ev);
    return true;
  } else {
    int32_t i32;
    if (!PyUpb_GetInt32(obj, &i32)) return false;
    if (upb_filedef_syntax(upb_enumdef_file(e)) == UPB_SYNTAX_PROTO2 &&
        !upb_enumdef_checknum(e, i32)) {
      PyErr_Format(PyExc_ValueError, "invalid enumerator %d", (int)i32);
      return false;
    }
    val->int32_val = i32;
    return true;
  }
}

bool PyUpb_PyToUpb(PyObject *obj, const upb_fielddef *f, upb_msgval *val,
                   upb_arena *arena) {
  switch (upb_fielddef_type(f)) {
    case UPB_TYPE_ENUM:
      return PyUpb_PyToUpbEnum(obj, upb_fielddef_enumsubdef(f), val);
    case UPB_TYPE_INT32:
      return PyUpb_GetInt32(obj, &val->int32_val);
    case UPB_TYPE_INT64:
      return PyUpb_GetInt64(obj, &val->int64_val);
    case UPB_TYPE_UINT32:
      return PyUpb_GetUint32(obj, &val->uint32_val);
    case UPB_TYPE_UINT64:
      return PyUpb_GetUint64(obj, &val->uint64_val);
    case UPB_TYPE_FLOAT:
      val->float_val = PyFloat_AsDouble(obj);
      return !PyErr_Occurred();
    case UPB_TYPE_DOUBLE:
      val->double_val = PyFloat_AsDouble(obj);
      return !PyErr_Occurred();
    case UPB_TYPE_BOOL:
      val->bool_val = PyLong_AsLong(obj);
      return !PyErr_Occurred();
    case UPB_TYPE_BYTES: {
      char *ptr;
      Py_ssize_t size;
      if (PyBytes_AsStringAndSize(obj, &ptr, &size) < 0) return false;
      *val = PyUpb_MaybeCopyString(ptr, size, arena);
      return true;
    }
    case UPB_TYPE_STRING: {
      Py_ssize_t size;
      const char *ptr;
      PyObject *unicode = NULL;
      if (PyBytes_Check(obj)) {
        unicode = obj = PyUnicode_FromEncodedObject(obj, "utf-8", NULL);
        if (!obj) return false;
      }
      ptr = PyUnicode_AsUTF8AndSize(obj, &size);
      if (PyErr_Occurred()) {
        Py_XDECREF(unicode);
        return false;
      }
      *val = PyUpb_MaybeCopyString(ptr, size, arena);
      Py_XDECREF(unicode);
      return true;
    }
    case UPB_TYPE_MESSAGE:
      PyErr_Format(
          PyExc_ValueError, "Message objects may not be assigned",
          upb_fielddef_type(f));
      return false;
    default:
      PyErr_Format(
          PyExc_SystemError, "Getting a value from a field of unknown type %d",
          upb_fielddef_type(f));
      return false;
  }
}

bool PyUpb_Message_IsEqual(const upb_msg *msg1, const upb_msg *msg2,
                           const upb_msgdef *m);

// -----------------------------------------------------------------------------
// Equal
// -----------------------------------------------------------------------------

bool PyUpb_ValueEq(upb_msgval val1, upb_msgval val2, const upb_fielddef *f) {
  switch (upb_fielddef_type(f)) {
    case UPB_TYPE_BOOL:
      return val1.bool_val == val2.bool_val;
    case UPB_TYPE_INT32:
    case UPB_TYPE_UINT32:
    case UPB_TYPE_ENUM:
      return val1.int32_val == val2.int32_val;
    case UPB_TYPE_INT64:
    case UPB_TYPE_UINT64:
      return val1.int64_val == val2.int64_val;
    case UPB_TYPE_FLOAT:
      return val1.float_val == val2.float_val;
    case UPB_TYPE_DOUBLE:
      return val1.double_val == val2.double_val;
    case UPB_TYPE_STRING:
    case UPB_TYPE_BYTES:
      return val1.str_val.size == val2.str_val.size &&
          memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == 0;
    case UPB_TYPE_MESSAGE:
      return PyUpb_Message_IsEqual(val1.msg_val, val2.msg_val,
                                   upb_fielddef_msgsubdef(f));
    default:
      return false;
  }
}

bool PyUpb_Map_IsEqual(const upb_map *map1, const upb_map *map2,
                       const upb_fielddef *f) {
  assert(upb_fielddef_ismap(f));
  if (map1 == map2) return true;

  size_t size1 = map1 ? upb_map_size(map1) : 0;
  size_t size2 = map2 ? upb_map_size(map2) : 0;
  if (size1 != size2) return false;
  if (size1 == 0) return true;

  const upb_msgdef *entry_m = upb_fielddef_msgsubdef(f);
  const upb_fielddef *val_f = upb_msgdef_field(entry_m, 1);
  size_t iter = UPB_MAP_BEGIN;

  while (upb_mapiter_next(map1, &iter)) {
    upb_msgval key = upb_mapiter_key(map1, iter);
    upb_msgval val1 = upb_mapiter_value(map1, iter);
    upb_msgval val2;
    if (!upb_map_get(map2, key, &val2)) return false;
    if (!PyUpb_ValueEq(val1, val2, val_f)) return false;
  }

  return true;
}

static bool PyUpb_ArrayElem_IsEqual(const upb_array *arr1,
                                    const upb_array *arr2, size_t i,
                                    const upb_fielddef *f) {
  assert(i < upb_array_size(arr1));
  assert(i < upb_array_size(arr2));
  upb_msgval val1 = upb_array_get(arr1, i);
  upb_msgval val2 = upb_array_get(arr2, i);
  return PyUpb_ValueEq(val1, val2, f);
}

bool PyUpb_Array_IsEqual(const upb_array *arr1, const upb_array *arr2,
                         const upb_fielddef *f) {
  assert(upb_fielddef_isseq(f) && !upb_fielddef_ismap(f));
  if (arr1 == arr2) return true;

  size_t n1 = arr1 ? upb_array_size(arr1) : 0;
  size_t n2 = arr2 ? upb_array_size(arr2) : 0;
  if (n1 != n2) return false;

  // Half the length rounded down.  Important: the empty list rounds to 0.
  size_t half = n1 / 2;

  // Search from the ends-in.  We expect differences to more quickly manifest
  // at the ends than in the middle.  If the length is odd we will miss the
  // middle element.
  for (size_t i = 0; i < half; i++) {
    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, i, f)) return false;
    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, n1 - 1 - i, f)) return false;
  }

  // For an odd-lengthed list, pick up the middle element.
  if (n1 & 1) {
    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, half, f)) return false;
  }

  return true;
}

bool PyUpb_Message_IsEqual(const upb_msg *msg1, const upb_msg *msg2,
                           const upb_msgdef *m) {
  if (msg1 == msg2) return true;
  if (upb_msg_extcount(msg1) != upb_msg_extcount(msg2)) return false;

  // Compare messages field-by-field.  This is slightly tricky, because while
  // we can iterate over normal fields in a predictable order, the extension
  // order is unpredictable and may be different between msg1 and msg2.
  // So we use the following strategy:
  //   1. Iterate over all msg1 fields (including extensions).
  //   2. For non-extension fields, we find the corresponding field by simply
  //      using upb_msg_next(msg2).  If the two messages have the same set of
  //      fields, this will yield the same field.
  //   3. For extension fields, we have to actually search for the corresponding
  //      field, which we do with upb_msg_get(msg2, ext_f1).
  //   4. Once iteration over msg1 is complete, we call upb_msg_next(msg2) one
  //      final time to verify that we have visited all of msg2's regular fields
  //      (we pass NULL for ext_dict so that iteration will *not* return
  //      extensions).
  //
  // We don't need to visit all of msg2's extensions, because we verified up
  // front that both messages have the same number of extensions.
  const upb_symtab* symtab = upb_filedef_symtab(upb_msgdef_file(m));
  const upb_fielddef *f1, *f2;
  upb_msgval val1, val2;
  size_t iter1 = UPB_MSG_BEGIN;
  size_t iter2 = UPB_MSG_BEGIN;
  while (upb_msg_next(msg1, m, symtab, &f1, &val1, &iter1)) {
    if (upb_fielddef_isextension(f1)) {
      val2 = upb_msg_get(msg2, f1);
    } else {
      if (!upb_msg_next(msg2, m, NULL, &f2, &val2, &iter2) || f1 != f2) {
        return false;
      }
    }

    if (upb_fielddef_ismap(f1)) {
      if (!PyUpb_Map_IsEqual(val1.map_val, val2.map_val, f1)) return false;
    } else if (upb_fielddef_isseq(f1)) {
      if (!PyUpb_Array_IsEqual(val1.array_val, val2.array_val, f1)) {
        return false;
      }
    } else {
      if (!PyUpb_ValueEq(val1, val2, f1)) return false;
    }
  }

  if (upb_msg_next(msg2, m, NULL, &f2, &val2, &iter2)) return false;

  size_t usize1, usize2;
  const char *uf1 = upb_msg_getunknown(msg1, &usize1);
  const char *uf2 = upb_msg_getunknown(msg2, &usize2);
  // 100 is arbitrary, we're trying to prevent stack overflow but it's not
  // obvious how deep we should allow here.
  return upb_Message_UnknownFieldsAreEqual(uf1, usize1, uf2, usize2, 100) ==
         kUpb_UnknownCompareResult_Equal;
}