Added map support.

python/BUILD

@@ -39,6 +39,8 @@ cc_binary(
         "descriptor_containers.h",
         "descriptor_pool.c",
         "descriptor_pool.h",
+        "map.c",
+        "map.h",
         "message.c",
         "message.h",
         "protobuf.c",

python/map.c

@@ -0,0 +1,487 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/map.h"
+
+#include "python/convert.h"
+#include "python/message.h"
+#include "python/protobuf.h"
+
+// -----------------------------------------------------------------------------
+// MapContainer
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD
+  PyObject* arena;
+  uintptr_t field;  // upb_fielddef*, low bit 1 == unset
+  union {
+    upb_map* map;      // when set, the data for this array.
+    PyObject* parent;  // when unset owning pointer to parent message.
+  };
+  int version;
+} PyUpb_MapContainer;
+
+static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map);
+
+static bool PyUpb_MapContainer_IsUnset(PyUpb_MapContainer* self) {
+  return self->field & 1;
+}
+
+static upb_map* PyUpb_MapContainer_GetIfWritable(PyUpb_MapContainer* self) {
+  return PyUpb_MapContainer_IsUnset(self) ? NULL : self->map;
+}
+
+static const upb_fielddef* PyUpb_MapContainer_GetField(
+    PyUpb_MapContainer* self) {
+  return (const upb_fielddef*)(self->field & ~(uintptr_t)1);
+}
+
+static void PyUpb_MapContainer_Dealloc(void* _self) {
+  PyUpb_MapContainer* self = _self;
+  Py_DECREF(self->arena);
+  if (PyUpb_MapContainer_IsUnset(self)) {
+    PyUpb_CMessage_CacheDelete(self->parent, PyUpb_MapContainer_GetField(self));
+    Py_DECREF(self->parent);
+  } else {
+    PyUpb_ObjCache_Delete(self->map);
+  }
+  PyUpb_Dealloc(_self);
+}
+
+PyTypeObject* PyUpb_MapContainer_GetClass(const upb_fielddef* f) {
+  assert(upb_fielddef_ismap(f));
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  return upb_fielddef_issubmsg(f) ? state->message_map_container_type
+                                  : state->scalar_map_container_type;
+}
+
+PyObject* PyUpb_MapContainer_NewUnset(PyObject* parent, const upb_fielddef* f,
+                                      PyObject* arena) {
+  PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
+  // We are GC because of the MutableMapping base class. Ideally we could
+  // implement them ourselves so we don't need a base so we don't need to be GC.
+  PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0);
+  map->arena = arena;
+  map->field = (uintptr_t)f | 1;
+  map->parent = parent;
+  map->version = 0;
+  Py_INCREF(arena);
+  Py_INCREF(parent);
+  return &map->ob_base;
+}
+
+void PyUpb_MapContainer_SwitchToSet(PyObject* _self, upb_map* map) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  PyUpb_ObjCache_Add(map, &self->ob_base);
+  Py_DECREF(self->parent);
+  self->map = map;  // Overwrites self->parent.
+  self->field = self->field & ~(uintptr_t)1;
+  assert(!PyUpb_MapContainer_IsUnset(self));
+}
+
+void PyUpb_MapContainer_Invalidate(PyObject* obj) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)obj;
+  self->version++;
+}
+
+static upb_map* PyUpb_MapContainer_AssureWritable(PyUpb_MapContainer* self) {
+  self->version++;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  if (map) return map;  // Already writable.
+
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  upb_arena* arena = PyUpb_Arena_Get(self->arena);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
+  map = upb_map_new(arena, upb_fielddef_type(key_f), upb_fielddef_type(val_f));
+  upb_msgval msgval = {.map_val = map};
+  PyUpb_CMessage_SetConcreteSubobj(self->parent, f, msgval);
+  PyUpb_MapContainer_SwitchToSet((PyObject*)self, map);
+  return map;
+}
+
+int PyUpb_MapContainer_AssignSubscript(PyObject* _self, PyObject* key,
+                                       PyObject* val) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_AssureWritable(self);
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
+  upb_arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_msgval u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return -1;
+
+  if (val) {
+    if (!PyUpb_PyToUpb(val, val_f, &u_val, arena)) return -1;
+    upb_map_set(map, u_key, u_val, arena);
+  } else {
+    if (!upb_map_delete(map, u_key)) {
+      PyErr_Format(PyExc_KeyError, "Key not present in map");
+      return -1;
+    }
+  }
+  return 0;
+}
+
+PyObject* PyUpb_MapContainer_Contains(PyObject* _self, PyObject* key) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  if (!map) Py_RETURN_FALSE;
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  upb_msgval u_key;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, NULL)) return NULL;
+  if (upb_map_get(map, u_key, NULL)) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+PyObject* PyUpb_MapContainer_Clear(PyObject* _self, PyObject* key) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  if (map) upb_map_clear(map);
+  Py_RETURN_NONE;
+}
+
+static PyObject* PyUpb_MapContainer_Get(PyObject* _self, PyObject* args,
+                                        PyObject* kwargs) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  static const char* kwlist[] = {"key", "default", NULL};
+  PyObject* key;
+  PyObject* default_value = NULL;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", (char**)kwlist, &key,
+                                   &default_value)) {
+    return NULL;
+  }
+
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
+  upb_arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_msgval u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
+  if (map && upb_map_get(map, u_key, &u_val)) {
+    return PyUpb_UpbToPy(u_val, val_f, self->arena);
+  } else {
+    if (default_value) {
+      Py_INCREF(default_value);
+      return default_value;
+    } else {
+      Py_RETURN_NONE;
+    }
+  }
+}
+
+static PyObject* PyUpb_MapContainer_GetEntryClass(PyObject* _self,
+                                                  PyObject* arg) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  return PyUpb_Descriptor_GetClass(entry_m);
+}
+
+Py_ssize_t PyUpb_MapContainer_Length(PyObject* _self) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  return map ? upb_map_size(map) : 0;
+}
+
+PyUpb_MapContainer* PyUpb_MapContainer_Check(PyObject* _self) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  if (!PyObject_TypeCheck(_self, state->message_map_container_type) &&
+      !PyObject_TypeCheck(_self, state->scalar_map_container_type)) {
+    PyErr_Format(PyExc_TypeError, "Expected protobuf map, but got %R", _self);
+    return NULL;
+  }
+  return (PyUpb_MapContainer*)_self;
+}
+
+int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
+                                     const upb_fielddef* f);
+
+static PyObject* PyUpb_MapContainer_MergeFrom(PyObject* _self, PyObject* _arg) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+
+  if (PyDict_Check(_arg)) {
+    return PyErr_Format(PyExc_AttributeError, "Merging of dict is not allowed");
+  }
+
+  if (PyUpb_CMessage_InitMapAttributes(_self, _arg, f) < 0) {
+    return NULL;
+  }
+
+  Py_RETURN_NONE;
+}
+
+static PyObject* PyUpb_MapContainer_Repr(PyObject* _self) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  PyObject* dict = PyDict_New();
+  if (map) {
+    size_t iter = UPB_MAP_BEGIN;
+    const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+    const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+    const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+    const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
+    while (upb_mapiter_next(map, &iter)) {
+      PyObject* key =
+          PyUpb_UpbToPy(upb_mapiter_key(map, iter), key_f, self->arena);
+      PyObject* val =
+          PyUpb_UpbToPy(upb_mapiter_value(map, iter), val_f, self->arena);
+      PyDict_SetItem(dict, key, val);
+      Py_DECREF(key);
+      Py_DECREF(val);
+    }
+  }
+  PyObject* repr = PyObject_Repr(dict);
+  Py_DECREF(dict);
+  return repr;
+}
+
+PyObject* PyUpb_MapContainer_Subscript(PyObject* _self, PyObject* key) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
+  upb_arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_msgval u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
+  if (!map || !upb_map_get(map, u_key, &u_val)) {
+    map = PyUpb_MapContainer_AssureWritable(self);
+    upb_arena* arena = PyUpb_Arena_Get(self->arena);
+    if (upb_fielddef_issubmsg(val_f)) {
+      u_val.msg_val = upb_msg_new(upb_fielddef_msgsubdef(val_f), arena);
+    } else {
+      memset(&u_val, 0, sizeof(u_val));
+    }
+    upb_map_set(map, u_key, u_val, arena);
+  }
+  return PyUpb_UpbToPy(u_val, val_f, self->arena);
+}
+
+PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_map* u_map,
+                                                const upb_fielddef* f,
+                                                PyObject* arena) {
+  PyObject* ret = PyUpb_ObjCache_Get(u_map);
+
+  if (!ret) {
+    PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
+    PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0);
+    map->arena = arena;
+    map->field = (uintptr_t)f;
+    map->map = u_map;
+    map->version = 0;
+    ret = &map->ob_base;
+    Py_INCREF(arena);
+    PyUpb_ObjCache_Add(u_map, ret);
+  }
+
+  return ret;
+}
+
+// -----------------------------------------------------------------------------
+// ScalarMapContainer
+// -----------------------------------------------------------------------------
+
+static PyMethodDef PyUpb_ScalarMapContainer_Methods[] = {
+    {"__contains__", PyUpb_MapContainer_Contains, METH_O,
+     "Tests whether a key is a member of the map."},
+    {"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+     "Makes a deep copy of the class." },
+   { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+     "Outputs picklable representation of the repeated field." },
+   */
+    {NULL, NULL},
+};
+
+static PyType_Slot PyUpb_ScalarMapContainer_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
+    {Py_mp_length, PyUpb_MapContainer_Length},
+    {Py_mp_subscript, PyUpb_MapContainer_Subscript},
+    {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
+    {Py_tp_methods, PyUpb_ScalarMapContainer_Methods},
+    {Py_tp_iter, PyUpb_MapIterator_New},
+    {Py_tp_repr, PyUpb_MapContainer_Repr},
+    {Py_tp_hash, PyObject_HashNotImplemented},
+    {0, NULL},
+};
+
+static PyType_Spec PyUpb_ScalarMapContainer_Spec = {
+    PYUPB_MODULE_NAME ".ScalarMapContainer",
+    sizeof(PyUpb_MapContainer),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    PyUpb_ScalarMapContainer_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// MessageMapContainer
+// -----------------------------------------------------------------------------
+
+static PyMethodDef PyUpb_MessageMapContainer_Methods[] = {
+    {"__contains__", PyUpb_MapContainer_Contains, METH_O,
+     "Tests whether the map contains this element."},
+    {"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"get_or_create", PyUpb_MapContainer_Subscript, METH_O,
+     "Alias for getitem, useful to make explicit that the map is mutated."},
+    {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+     "Makes a deep copy of the class." },
+   { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+     "Outputs picklable representation of the repeated field." },
+   */
+    {NULL, NULL},
+};
+
+static PyType_Slot PyUpb_MessageMapContainer_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
+    {Py_mp_length, PyUpb_MapContainer_Length},
+    {Py_mp_subscript, PyUpb_MapContainer_Subscript},
+    {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
+    {Py_tp_methods, PyUpb_MessageMapContainer_Methods},
+    {Py_tp_iter, PyUpb_MapIterator_New},
+    {Py_tp_repr, PyUpb_MapContainer_Repr},
+    {Py_tp_hash, PyObject_HashNotImplemented},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_MessageMapContainer_Spec = {
+    PYUPB_MODULE_NAME ".MessageMapContainer", sizeof(PyUpb_MapContainer), 0,
+    Py_TPFLAGS_DEFAULT, PyUpb_MessageMapContainer_Slots};
+
+// -----------------------------------------------------------------------------
+// MapIterator
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD
+  PyUpb_MapContainer* map;  // We own a reference.
+  size_t iter;
+  int version;
+} PyUpb_MapIterator;
+
+static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  PyUpb_MapIterator* iter =
+      (void*)PyType_GenericAlloc(state->map_iterator_type, 0);
+  iter->map = map;
+  iter->iter = UPB_MAP_BEGIN;
+  iter->version = map->version;
+  Py_INCREF(map);
+  return &iter->ob_base;
+}
+
+static void PyUpb_MapIterator_Dealloc(void* _self) {
+  PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
+  Py_DECREF(&self->map->ob_base);
+  PyUpb_Dealloc(_self);
+}
+
+PyObject* PyUpb_MapIterator_IterNext(PyObject* _self) {
+  PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
+  if (self->version != self->map->version) {
+    return PyErr_Format(PyExc_RuntimeError, "Map modified during iteration.");
+  }
+  upb_map* map = PyUpb_MapContainer_GetIfWritable(self->map);
+  if (!map) return NULL;
+  if (!upb_mapiter_next(map, &self->iter)) return NULL;
+  upb_msgval key = upb_mapiter_key(map, self->iter);
+  const upb_fielddef* f = PyUpb_MapContainer_GetField(self->map);
+  const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
+  const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
+  return PyUpb_UpbToPy(key, key_f, self->map->arena);
+}
+
+static PyType_Slot PyUpb_MapIterator_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapIterator_Dealloc},
+    {Py_tp_iter, PyObject_SelfIter},
+    {Py_tp_iternext, PyUpb_MapIterator_IterNext},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_MapIterator_Spec = {
+    PYUPB_MODULE_NAME ".MapIterator", sizeof(PyUpb_MapIterator), 0,
+    Py_TPFLAGS_DEFAULT, PyUpb_MapIterator_Slots};
+
+// -----------------------------------------------------------------------------
+// Top Level
+// -----------------------------------------------------------------------------
+
+static PyObject* GetMutableMappingBase(void) {
+  PyObject* collections = NULL;
+  PyObject* mapping = NULL;
+  PyObject* bases = NULL;
+  if ((collections = PyImport_ImportModule("collections.abc")) &&
+      (mapping = PyObject_GetAttrString(collections, "MutableMapping"))) {
+    bases = Py_BuildValue("(O)", mapping);
+  }
+  Py_XDECREF(collections);
+  Py_XDECREF(mapping);
+  return bases;
+}
+
+bool PyUpb_Map_Init(PyObject* m) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m);
+  PyObject* bases = GetMutableMappingBase();
+  if (!bases) return false;
+
+  state->message_map_container_type =
+      PyUpb_AddClassWithBases(m, &PyUpb_MessageMapContainer_Spec, bases);
+  state->scalar_map_container_type =
+      PyUpb_AddClassWithBases(m, &PyUpb_ScalarMapContainer_Spec, bases);
+  state->map_iterator_type = PyUpb_AddClass(m, &PyUpb_MapIterator_Spec);
+
+  Py_DECREF(bases);
+
+  return state->message_map_container_type &&
+         state->scalar_map_container_type && state->map_iterator_type;
+}

python/map.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_MAP_H__
+#define PYUPB_MAP_H__
+
+#include <stdbool.h>
+
+#include "python/python.h"
+#include "upb/def.h"
+
+PyObject* PyUpb_MapContainer_NewUnset(PyObject* parent, const upb_fielddef* f,
+                                      PyObject* arena);
+PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_map* u_map,
+                                                const upb_fielddef* f,
+                                                PyObject* arena);
+void PyUpb_MapContainer_SwitchToSet(PyObject* self, upb_map* map);
+int PyUpb_MapContainer_AssignSubscript(PyObject* self, PyObject* key,
+                                       PyObject* val);
+void PyUpb_MapContainer_Invalidate(PyObject* obj);
+
+bool PyUpb_Map_Init(PyObject* m);
+
+#endif  // PYUPB_MAP_H__

python/message.c

@@ -29,6 +29,7 @@
 
 #include "python/convert.h"
 #include "python/descriptor.h"
+#include "python/map.h"
 #include "python/repeated.h"
 #include "upb/def.h"
 #include "upb/reflection.h"
@@ -286,7 +287,7 @@ static bool PyUpb_CMessage_InitMessageMapEntry(PyObject* dst, PyObject* src) {
   return true;
 }
 
-static int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
+int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
                                      const upb_fielddef* f) {
   const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
   const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
@@ -327,13 +328,10 @@ void PyUpb_CMessage_AssureWritable(PyUpb_CMessage* self);
 static bool PyUpb_CMessage_InitMapAttribute(PyObject* _self, PyObject* name,
                                             const upb_fielddef* f,
                                             PyObject* value) {
-  // TODO(haberman): disabled until map container is in.
+  PyObject* map = PyUpb_CMessage_GetAttr(_self, name);
-  // PyObject* map = PyUpb_CMessage_GetAttr(_self, name);
+  int ok = PyUpb_CMessage_InitMapAttributes(map, value, f);
-  // int ok = PyUpb_CMessage_InitMapAttributes(map, value, f);
+  Py_DECREF(map);
-  // Py_DECREF(map);
+  return ok >= 0;
-  // return ok >= 0;
-  PyErr_SetString(PyExc_NotImplementedError, "map init");
-  return false;
 }
 
 static bool PyUpb_CMessage_InitRepeatedAttribute(PyObject* _self,
@@ -598,8 +596,7 @@ static void PyUpb_CMessage_SyncSubobjs(PyUpb_CMessage* self) {
     PyUpb_WeakMap_DeleteIter(subobj_map, &iter);
     if (upb_fielddef_ismap(f)) {
       if (!msgval.map_val) continue;
-      // TODO(haberman): re-enable when maps are checked in.
+      PyUpb_MapContainer_SwitchToSet(obj, (upb_map*)msgval.map_val);
-      // PyUpb_MapContainer_SwitchToSet(obj, (upb_map*)msgval.map_val);
     } else if (upb_fielddef_isseq(f)) {
       if (!msgval.array_val) continue;
       PyUpb_RepeatedContainer_Reify(obj, (upb_array*)msgval.array_val);
@@ -730,10 +727,7 @@ PyObject* PyUpb_CMessage_GetUnsetWrapper(PyUpb_CMessage* self,
   if (subobj) return subobj;
 
   if (upb_fielddef_ismap(field)) {
-    // TODO(haberman): re-enable when maps are checked in.
+    subobj = PyUpb_MapContainer_NewUnset(_self, field, self->arena);
-    // subobj = PyUpb_MapContainer_NewUnset(_self, field, self->arena);
-    PyErr_SetString(PyExc_NotImplementedError, "unset map");
-    return NULL;
   } else if (upb_fielddef_isseq(field)) {
     subobj = PyUpb_RepeatedContainer_NewUnset(_self, field, self->arena);
   } else {
@@ -751,12 +745,8 @@ PyObject* PyUpb_CMessage_GetPresentWrapper(PyUpb_CMessage* self,
   upb_mutmsgval mutval =
       upb_msg_mutable(self->ptr.msg, field, PyUpb_Arena_Get(self->arena));
   if (upb_fielddef_ismap(field)) {
-    // TODO(haberman): re-enable when maps are checked in.
+    return PyUpb_MapContainer_GetOrCreateWrapper(mutval.map, field,
-    // return PyUpb_MapContainer_GetOrCreateWrapper(mutval.map, field,
+                                                 self->arena);
-    //                                              self->arena);
-    (void)mutval;
-    PyErr_SetString(PyExc_NotImplementedError, "access map");
-    return NULL;
   } else {
     return PyUpb_RepeatedContainer_GetOrCreateWrapper(mutval.array, field,
                                                       self->arena);
@@ -1036,7 +1026,8 @@ static PyObject* PyUpb_CMessage_Clear(PyUpb_CMessage* self, PyObject* args) {
 
 static void PyUpb_CMessage_AbandonField(PyUpb_CMessage* self,
                                         const upb_fielddef* f) {
-  if (self->unset_subobj_map && upb_fielddef_issubmsg(f)) {
+  if (self->unset_subobj_map && upb_fielddef_issubmsg(f) &&
+      !upb_fielddef_isseq(f)) {
     PyObject* sub = PyUpb_WeakMap_Get(self->unset_subobj_map, f);
     if (sub) {
       PyUpb_CMessage_AssureWritable((PyUpb_CMessage*)sub);
@@ -1089,8 +1080,7 @@ static PyObject* PyUpb_CMessage_ClearField(PyUpb_CMessage* self,
       obj = PyUpb_ObjCache_Get(msgval.map_val);
     }
     if (obj) {
-      // TODO(haberman) add when maps are checked in.
+      PyUpb_MapContainer_Invalidate(obj);
-      // PyUpb_MapContainer_Invalidate(obj);
       Py_DECREF(obj);
     }
   }

python/pb_unit_tests/message_test_wrapper.py

@@ -59,41 +59,14 @@ message_test.Proto2Test.testMergeFromExtensions.__unittest_expecting_failure__ =
 message_test.Proto2Test.testParsingMerge.__unittest_expecting_failure__ = True
 message_test.Proto2Test.testPickleIncompleteProto.__unittest_expecting_failure__ = True
 message_test.Proto2Test.testPythonicInit.__unittest_expecting_failure__ = True
-message_test.Proto2Test.testUnknownEnumMap.__unittest_expecting_failure__ = True
 message_test.Proto2Test.test_documentation.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testCopyFromBadType.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testIntegerMapWithLongs.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMapAssignmentCausesPresence.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMapAssignmentCausesPresenceForSubmessages.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapByteSize.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapConstruction.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapDelete.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMapDeterministicSerialization.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapEntryAlwaysSerialized.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapFieldRaisesCorrectError.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapFindInitializationErrorsSmokeTest.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapGet.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapItems.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapIterInvalidatedByClearField.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapIteration.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapIterationClearMessage.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMapMergeFrom.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapMessageFieldConstruction.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapScalarFieldConstruction.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapValidAfterFieldCleared.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapsAreMapping.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMapsCompare.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMergeFrom.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMergeFromBadType.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMessageMap.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMessageMapItemValidAfterTopMessageCleared.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testMessageMapValidAfterFieldCleared.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testModifyMapWhileIterating.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testNestedMessageMapItemDelete.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testScalarMap.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testScalarMapDefaults.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testStringUnicodeConversionInMap.__unittest_expecting_failure__ = True
-message_test.Proto3Test.testSubmessageMap.__unittest_expecting_failure__ = True
 message_test.OversizeProtosTest.testAssertOversizeProto.__unittest_expecting_failure__ = True
 message_test.OversizeProtosTest.testSucceedOversizeProto.__unittest_expecting_failure__ = True
 

python/pb_unit_tests/unknown_fields_test_wrapper.py

@@ -35,7 +35,6 @@ unknown_fields_test.UnknownFieldsAccessorsTest.testSubUnknownFields.__unittest_e
 unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownExtensions.__unittest_expecting_failure__ = True
 unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownField.__unittest_expecting_failure__ = True
 unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownFieldsNoMemoryLeak.__unittest_expecting_failure__ = True
-unknown_fields_test.UnknownFieldsTest.testDiscardUnknownFields.__unittest_expecting_failure__ = True
 unknown_fields_test.UnknownFieldsTest.testEquals.__unittest_expecting_failure__ = True
 unknown_fields_test.UnknownFieldsTest.testSerializeMessageSetWireFormatUnknownExtension.__unittest_expecting_failure__ = True
 

python/protobuf.c

@@ -30,6 +30,7 @@
 #include "python/descriptor.h"
 #include "python/descriptor_containers.h"
 #include "python/descriptor_pool.h"
+#include "python/map.h"
 #include "python/message.h"
 #include "python/repeated.h"
 
@@ -254,8 +255,16 @@ static const char *PyUpb_GetClassName(PyType_Spec *spec) {
 PyTypeObject *PyUpb_AddClass(PyObject *m, PyType_Spec *spec) {
   PyObject *type = PyType_FromSpec(spec);
   const char *name = PyUpb_GetClassName(spec);
-  return type && PyModule_AddObject(m, name, type) == 0 ? (PyTypeObject *)type
+  int result = PyModule_AddObject(m, name, type);
-                                                        : NULL;
+  return type && result == 0 ? (PyTypeObject*)type : NULL;
+}
+
+PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec,
+                                      PyObject* bases) {
+  PyObject* type = PyType_FromSpecWithBases(spec, bases);
+  const char* name = PyUpb_GetClassName(spec);
+  int result = PyModule_AddObject(m, name, type);
+  return type && result == 0 ? (PyTypeObject*)type : NULL;
 }
 
 const char *PyUpb_GetStrData(PyObject *obj) {
@@ -290,7 +299,7 @@ PyMODINIT_FUNC PyInit__message(void) {
   state->obj_cache = PyUpb_WeakMap_New();
 
   if (!PyUpb_InitDescriptorContainers(m) || !PyUpb_InitDescriptorPool(m) ||
-      !PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) ||
+      !PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) || !PyUpb_Map_Init(m) ||
       !PyUpb_InitMessage(m) || !PyUpb_Repeated_Init(m)) {
     Py_DECREF(m);
     return NULL;

python/protobuf.h

@@ -64,6 +64,11 @@ typedef struct {
   // From descriptor_pool.c
   PyTypeObject *descriptor_pool_type;
 
+  // From map.c
+  PyTypeObject* map_iterator_type;
+  PyTypeObject* message_map_container_type;
+  PyTypeObject* scalar_map_container_type;
+
   // From message.c
   PyObject *decode_error_class;
   PyObject* descriptor_string;
@@ -165,6 +170,11 @@ PyTypeObject *AddObject(PyObject *m, const char *name, PyType_Spec *spec);
 // Creates a Python type from `spec` and adds it to the given module `m`.
 PyTypeObject *PyUpb_AddClass(PyObject *m, PyType_Spec *spec);
 
+// Like PyUpb_AddClass(), but allows you to specify a tuple of base classes
+// in `bases`.
+PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec,
+                                      PyObject* bases);
+
 // A function that implements the tp_new slot for types that we do not allow
 // users to create directly. This will immediately fail with an error message.
 PyObject *PyUpb_Forbidden_New(PyObject *cls, PyObject *args, PyObject *kwds);

upb/msg.c

@@ -285,6 +285,7 @@ static int _upb_mapsorter_cmpstr(const void *_a, const void *_b) {
   return a.size - b.size;
 }
 
+#include <stdio.h>
 bool _upb_mapsorter_pushmap(_upb_mapsorter *s, upb_descriptortype_t key_type,
                             const upb_map *map, _upb_sortedmap *sorted) {
   int map_size = _upb_map_size(map);
@@ -341,6 +342,7 @@ bool _upb_mapsorter_pushmap(_upb_mapsorter *s, upb_descriptortype_t key_type,
       compar = _upb_mapsorter_cmpbool;
       break;
     case UPB_DESCRIPTOR_TYPE_STRING:
+    case UPB_DESCRIPTOR_TYPE_BYTES:
       compar = _upb_mapsorter_cmpstr;
       break;
     default: