Added map support.

pull/13171/head
Joshua Haberman 3 years ago
parent 10597019ba
commit bf74b3e7fd
  1. 2
      python/BUILD
  2. 487
      python/map.c
  3. 48
      python/map.h
  4. 38
      python/message.c
  5. 27
      python/pb_unit_tests/message_test_wrapper.py
  6. 1
      python/pb_unit_tests/unknown_fields_test_wrapper.py
  7. 15
      python/protobuf.c
  8. 10
      python/protobuf.h
  9. 2
      upb/msg.c

@ -39,6 +39,8 @@ cc_binary(
"descriptor_containers.h",
"descriptor_pool.c",
"descriptor_pool.h",
"map.c",
"map.h",
"message.c",
"message.h",
"protobuf.c",

@ -0,0 +1,487 @@
/*
* Copyright (c) 2009-2021, Google LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Google LLC nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "python/map.h"
#include "python/convert.h"
#include "python/message.h"
#include "python/protobuf.h"
// -----------------------------------------------------------------------------
// MapContainer
// -----------------------------------------------------------------------------
typedef struct {
PyObject_HEAD
PyObject* arena;
uintptr_t field; // upb_fielddef*, low bit 1 == unset
union {
upb_map* map; // when set, the data for this array.
PyObject* parent; // when unset owning pointer to parent message.
};
int version;
} PyUpb_MapContainer;
static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map);
static bool PyUpb_MapContainer_IsUnset(PyUpb_MapContainer* self) {
return self->field & 1;
}
static upb_map* PyUpb_MapContainer_GetIfWritable(PyUpb_MapContainer* self) {
return PyUpb_MapContainer_IsUnset(self) ? NULL : self->map;
}
static const upb_fielddef* PyUpb_MapContainer_GetField(
PyUpb_MapContainer* self) {
return (const upb_fielddef*)(self->field & ~(uintptr_t)1);
}
static void PyUpb_MapContainer_Dealloc(void* _self) {
PyUpb_MapContainer* self = _self;
Py_DECREF(self->arena);
if (PyUpb_MapContainer_IsUnset(self)) {
PyUpb_CMessage_CacheDelete(self->parent, PyUpb_MapContainer_GetField(self));
Py_DECREF(self->parent);
} else {
PyUpb_ObjCache_Delete(self->map);
}
PyUpb_Dealloc(_self);
}
PyTypeObject* PyUpb_MapContainer_GetClass(const upb_fielddef* f) {
assert(upb_fielddef_ismap(f));
PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
return upb_fielddef_issubmsg(f) ? state->message_map_container_type
: state->scalar_map_container_type;
}
PyObject* PyUpb_MapContainer_NewUnset(PyObject* parent, const upb_fielddef* f,
PyObject* arena) {
PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
// We are GC because of the MutableMapping base class. Ideally we could
// implement them ourselves so we don't need a base so we don't need to be GC.
PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0);
map->arena = arena;
map->field = (uintptr_t)f | 1;
map->parent = parent;
map->version = 0;
Py_INCREF(arena);
Py_INCREF(parent);
return &map->ob_base;
}
void PyUpb_MapContainer_SwitchToSet(PyObject* _self, upb_map* map) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
PyUpb_ObjCache_Add(map, &self->ob_base);
Py_DECREF(self->parent);
self->map = map; // Overwrites self->parent.
self->field = self->field & ~(uintptr_t)1;
assert(!PyUpb_MapContainer_IsUnset(self));
}
void PyUpb_MapContainer_Invalidate(PyObject* obj) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)obj;
self->version++;
}
static upb_map* PyUpb_MapContainer_AssureWritable(PyUpb_MapContainer* self) {
self->version++;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
if (map) return map; // Already writable.
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
upb_arena* arena = PyUpb_Arena_Get(self->arena);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
map = upb_map_new(arena, upb_fielddef_type(key_f), upb_fielddef_type(val_f));
upb_msgval msgval = {.map_val = map};
PyUpb_CMessage_SetConcreteSubobj(self->parent, f, msgval);
PyUpb_MapContainer_SwitchToSet((PyObject*)self, map);
return map;
}
int PyUpb_MapContainer_AssignSubscript(PyObject* _self, PyObject* key,
PyObject* val) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_AssureWritable(self);
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
upb_arena* arena = PyUpb_Arena_Get(self->arena);
upb_msgval u_key, u_val;
if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return -1;
if (val) {
if (!PyUpb_PyToUpb(val, val_f, &u_val, arena)) return -1;
upb_map_set(map, u_key, u_val, arena);
} else {
if (!upb_map_delete(map, u_key)) {
PyErr_Format(PyExc_KeyError, "Key not present in map");
return -1;
}
}
return 0;
}
PyObject* PyUpb_MapContainer_Contains(PyObject* _self, PyObject* key) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
if (!map) Py_RETURN_FALSE;
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
upb_msgval u_key;
if (!PyUpb_PyToUpb(key, key_f, &u_key, NULL)) return NULL;
if (upb_map_get(map, u_key, NULL)) {
Py_RETURN_TRUE;
} else {
Py_RETURN_FALSE;
}
}
PyObject* PyUpb_MapContainer_Clear(PyObject* _self, PyObject* key) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
if (map) upb_map_clear(map);
Py_RETURN_NONE;
}
static PyObject* PyUpb_MapContainer_Get(PyObject* _self, PyObject* args,
PyObject* kwargs) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
static const char* kwlist[] = {"key", "default", NULL};
PyObject* key;
PyObject* default_value = NULL;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", (char**)kwlist, &key,
&default_value)) {
return NULL;
}
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
upb_arena* arena = PyUpb_Arena_Get(self->arena);
upb_msgval u_key, u_val;
if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
if (map && upb_map_get(map, u_key, &u_val)) {
return PyUpb_UpbToPy(u_val, val_f, self->arena);
} else {
if (default_value) {
Py_INCREF(default_value);
return default_value;
} else {
Py_RETURN_NONE;
}
}
}
static PyObject* PyUpb_MapContainer_GetEntryClass(PyObject* _self,
PyObject* arg) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
return PyUpb_Descriptor_GetClass(entry_m);
}
Py_ssize_t PyUpb_MapContainer_Length(PyObject* _self) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
return map ? upb_map_size(map) : 0;
}
PyUpb_MapContainer* PyUpb_MapContainer_Check(PyObject* _self) {
PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
if (!PyObject_TypeCheck(_self, state->message_map_container_type) &&
!PyObject_TypeCheck(_self, state->scalar_map_container_type)) {
PyErr_Format(PyExc_TypeError, "Expected protobuf map, but got %R", _self);
return NULL;
}
return (PyUpb_MapContainer*)_self;
}
int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
const upb_fielddef* f);
static PyObject* PyUpb_MapContainer_MergeFrom(PyObject* _self, PyObject* _arg) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
if (PyDict_Check(_arg)) {
return PyErr_Format(PyExc_AttributeError, "Merging of dict is not allowed");
}
if (PyUpb_CMessage_InitMapAttributes(_self, _arg, f) < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* PyUpb_MapContainer_Repr(PyObject* _self) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
PyObject* dict = PyDict_New();
if (map) {
size_t iter = UPB_MAP_BEGIN;
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
while (upb_mapiter_next(map, &iter)) {
PyObject* key =
PyUpb_UpbToPy(upb_mapiter_key(map, iter), key_f, self->arena);
PyObject* val =
PyUpb_UpbToPy(upb_mapiter_value(map, iter), val_f, self->arena);
PyDict_SetItem(dict, key, val);
Py_DECREF(key);
Py_DECREF(val);
}
}
PyObject* repr = PyObject_Repr(dict);
Py_DECREF(dict);
return repr;
}
PyObject* PyUpb_MapContainer_Subscript(PyObject* _self, PyObject* key) {
PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
upb_map* map = PyUpb_MapContainer_GetIfWritable(self);
const upb_fielddef* f = PyUpb_MapContainer_GetField(self);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
upb_arena* arena = PyUpb_Arena_Get(self->arena);
upb_msgval u_key, u_val;
if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
if (!map || !upb_map_get(map, u_key, &u_val)) {
map = PyUpb_MapContainer_AssureWritable(self);
upb_arena* arena = PyUpb_Arena_Get(self->arena);
if (upb_fielddef_issubmsg(val_f)) {
u_val.msg_val = upb_msg_new(upb_fielddef_msgsubdef(val_f), arena);
} else {
memset(&u_val, 0, sizeof(u_val));
}
upb_map_set(map, u_key, u_val, arena);
}
return PyUpb_UpbToPy(u_val, val_f, self->arena);
}
PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_map* u_map,
const upb_fielddef* f,
PyObject* arena) {
PyObject* ret = PyUpb_ObjCache_Get(u_map);
if (!ret) {
PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0);
map->arena = arena;
map->field = (uintptr_t)f;
map->map = u_map;
map->version = 0;
ret = &map->ob_base;
Py_INCREF(arena);
PyUpb_ObjCache_Add(u_map, ret);
}
return ret;
}
// -----------------------------------------------------------------------------
// ScalarMapContainer
// -----------------------------------------------------------------------------
static PyMethodDef PyUpb_ScalarMapContainer_Methods[] = {
{"__contains__", PyUpb_MapContainer_Contains, METH_O,
"Tests whether a key is a member of the map."},
{"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
"Removes all elements from the map."},
{"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
"Gets the value for the given key if present, or otherwise a default"},
{"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
"Return the class used to build Entries of (key, value) pairs."},
{"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
"Merges a map into the current map."},
/*
{ "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
"Makes a deep copy of the class." },
{ "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
"Outputs picklable representation of the repeated field." },
*/
{NULL, NULL},
};
static PyType_Slot PyUpb_ScalarMapContainer_Slots[] = {
{Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
{Py_mp_length, PyUpb_MapContainer_Length},
{Py_mp_subscript, PyUpb_MapContainer_Subscript},
{Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
{Py_tp_methods, PyUpb_ScalarMapContainer_Methods},
{Py_tp_iter, PyUpb_MapIterator_New},
{Py_tp_repr, PyUpb_MapContainer_Repr},
{Py_tp_hash, PyObject_HashNotImplemented},
{0, NULL},
};
static PyType_Spec PyUpb_ScalarMapContainer_Spec = {
PYUPB_MODULE_NAME ".ScalarMapContainer",
sizeof(PyUpb_MapContainer),
0,
Py_TPFLAGS_DEFAULT,
PyUpb_ScalarMapContainer_Slots,
};
// -----------------------------------------------------------------------------
// MessageMapContainer
// -----------------------------------------------------------------------------
static PyMethodDef PyUpb_MessageMapContainer_Methods[] = {
{"__contains__", PyUpb_MapContainer_Contains, METH_O,
"Tests whether the map contains this element."},
{"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
"Removes all elements from the map."},
{"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
"Gets the value for the given key if present, or otherwise a default"},
{"get_or_create", PyUpb_MapContainer_Subscript, METH_O,
"Alias for getitem, useful to make explicit that the map is mutated."},
{"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
"Return the class used to build Entries of (key, value) pairs."},
{"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
"Merges a map into the current map."},
/*
{ "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
"Makes a deep copy of the class." },
{ "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
"Outputs picklable representation of the repeated field." },
*/
{NULL, NULL},
};
static PyType_Slot PyUpb_MessageMapContainer_Slots[] = {
{Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
{Py_mp_length, PyUpb_MapContainer_Length},
{Py_mp_subscript, PyUpb_MapContainer_Subscript},
{Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
{Py_tp_methods, PyUpb_MessageMapContainer_Methods},
{Py_tp_iter, PyUpb_MapIterator_New},
{Py_tp_repr, PyUpb_MapContainer_Repr},
{Py_tp_hash, PyObject_HashNotImplemented},
{0, NULL}};
static PyType_Spec PyUpb_MessageMapContainer_Spec = {
PYUPB_MODULE_NAME ".MessageMapContainer", sizeof(PyUpb_MapContainer), 0,
Py_TPFLAGS_DEFAULT, PyUpb_MessageMapContainer_Slots};
// -----------------------------------------------------------------------------
// MapIterator
// -----------------------------------------------------------------------------
typedef struct {
PyObject_HEAD
PyUpb_MapContainer* map; // We own a reference.
size_t iter;
int version;
} PyUpb_MapIterator;
static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map) {
PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
PyUpb_MapIterator* iter =
(void*)PyType_GenericAlloc(state->map_iterator_type, 0);
iter->map = map;
iter->iter = UPB_MAP_BEGIN;
iter->version = map->version;
Py_INCREF(map);
return &iter->ob_base;
}
static void PyUpb_MapIterator_Dealloc(void* _self) {
PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
Py_DECREF(&self->map->ob_base);
PyUpb_Dealloc(_self);
}
PyObject* PyUpb_MapIterator_IterNext(PyObject* _self) {
PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
if (self->version != self->map->version) {
return PyErr_Format(PyExc_RuntimeError, "Map modified during iteration.");
}
upb_map* map = PyUpb_MapContainer_GetIfWritable(self->map);
if (!map) return NULL;
if (!upb_mapiter_next(map, &self->iter)) return NULL;
upb_msgval key = upb_mapiter_key(map, self->iter);
const upb_fielddef* f = PyUpb_MapContainer_GetField(self->map);
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* key_f = upb_msgdef_field(entry_m, 0);
return PyUpb_UpbToPy(key, key_f, self->map->arena);
}
static PyType_Slot PyUpb_MapIterator_Slots[] = {
{Py_tp_dealloc, PyUpb_MapIterator_Dealloc},
{Py_tp_iter, PyObject_SelfIter},
{Py_tp_iternext, PyUpb_MapIterator_IterNext},
{0, NULL}};
static PyType_Spec PyUpb_MapIterator_Spec = {
PYUPB_MODULE_NAME ".MapIterator", sizeof(PyUpb_MapIterator), 0,
Py_TPFLAGS_DEFAULT, PyUpb_MapIterator_Slots};
// -----------------------------------------------------------------------------
// Top Level
// -----------------------------------------------------------------------------
static PyObject* GetMutableMappingBase(void) {
PyObject* collections = NULL;
PyObject* mapping = NULL;
PyObject* bases = NULL;
if ((collections = PyImport_ImportModule("collections.abc")) &&
(mapping = PyObject_GetAttrString(collections, "MutableMapping"))) {
bases = Py_BuildValue("(O)", mapping);
}
Py_XDECREF(collections);
Py_XDECREF(mapping);
return bases;
}
bool PyUpb_Map_Init(PyObject* m) {
PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m);
PyObject* bases = GetMutableMappingBase();
if (!bases) return false;
state->message_map_container_type =
PyUpb_AddClassWithBases(m, &PyUpb_MessageMapContainer_Spec, bases);
state->scalar_map_container_type =
PyUpb_AddClassWithBases(m, &PyUpb_ScalarMapContainer_Spec, bases);
state->map_iterator_type = PyUpb_AddClass(m, &PyUpb_MapIterator_Spec);
Py_DECREF(bases);
return state->message_map_container_type &&
state->scalar_map_container_type && state->map_iterator_type;
}

@ -0,0 +1,48 @@
/*
* Copyright (c) 2009-2021, Google LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Google LLC nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef PYUPB_MAP_H__
#define PYUPB_MAP_H__
#include <stdbool.h>
#include "python/python.h"
#include "upb/def.h"
PyObject* PyUpb_MapContainer_NewUnset(PyObject* parent, const upb_fielddef* f,
PyObject* arena);
PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_map* u_map,
const upb_fielddef* f,
PyObject* arena);
void PyUpb_MapContainer_SwitchToSet(PyObject* self, upb_map* map);
int PyUpb_MapContainer_AssignSubscript(PyObject* self, PyObject* key,
PyObject* val);
void PyUpb_MapContainer_Invalidate(PyObject* obj);
bool PyUpb_Map_Init(PyObject* m);
#endif // PYUPB_MAP_H__

@ -29,6 +29,7 @@
#include "python/convert.h"
#include "python/descriptor.h"
#include "python/map.h"
#include "python/repeated.h"
#include "upb/def.h"
#include "upb/reflection.h"
@ -286,8 +287,8 @@ static bool PyUpb_CMessage_InitMessageMapEntry(PyObject* dst, PyObject* src) {
return true;
}
static int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
const upb_fielddef* f) {
int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
const upb_fielddef* f) {
const upb_msgdef* entry_m = upb_fielddef_msgsubdef(f);
const upb_fielddef* val_f = upb_msgdef_field(entry_m, 1);
PyObject* it = NULL;
@ -327,13 +328,10 @@ void PyUpb_CMessage_AssureWritable(PyUpb_CMessage* self);
static bool PyUpb_CMessage_InitMapAttribute(PyObject* _self, PyObject* name,
const upb_fielddef* f,
PyObject* value) {
// TODO(haberman): disabled until map container is in.
// PyObject* map = PyUpb_CMessage_GetAttr(_self, name);
// int ok = PyUpb_CMessage_InitMapAttributes(map, value, f);
// Py_DECREF(map);
// return ok >= 0;
PyErr_SetString(PyExc_NotImplementedError, "map init");
return false;
PyObject* map = PyUpb_CMessage_GetAttr(_self, name);
int ok = PyUpb_CMessage_InitMapAttributes(map, value, f);
Py_DECREF(map);
return ok >= 0;
}
static bool PyUpb_CMessage_InitRepeatedAttribute(PyObject* _self,
@ -598,8 +596,7 @@ static void PyUpb_CMessage_SyncSubobjs(PyUpb_CMessage* self) {
PyUpb_WeakMap_DeleteIter(subobj_map, &iter);
if (upb_fielddef_ismap(f)) {
if (!msgval.map_val) continue;
// TODO(haberman): re-enable when maps are checked in.
// PyUpb_MapContainer_SwitchToSet(obj, (upb_map*)msgval.map_val);
PyUpb_MapContainer_SwitchToSet(obj, (upb_map*)msgval.map_val);
} else if (upb_fielddef_isseq(f)) {
if (!msgval.array_val) continue;
PyUpb_RepeatedContainer_Reify(obj, (upb_array*)msgval.array_val);
@ -730,10 +727,7 @@ PyObject* PyUpb_CMessage_GetUnsetWrapper(PyUpb_CMessage* self,
if (subobj) return subobj;
if (upb_fielddef_ismap(field)) {
// TODO(haberman): re-enable when maps are checked in.
// subobj = PyUpb_MapContainer_NewUnset(_self, field, self->arena);
PyErr_SetString(PyExc_NotImplementedError, "unset map");
return NULL;
subobj = PyUpb_MapContainer_NewUnset(_self, field, self->arena);
} else if (upb_fielddef_isseq(field)) {
subobj = PyUpb_RepeatedContainer_NewUnset(_self, field, self->arena);
} else {
@ -751,12 +745,8 @@ PyObject* PyUpb_CMessage_GetPresentWrapper(PyUpb_CMessage* self,
upb_mutmsgval mutval =
upb_msg_mutable(self->ptr.msg, field, PyUpb_Arena_Get(self->arena));
if (upb_fielddef_ismap(field)) {
// TODO(haberman): re-enable when maps are checked in.
// return PyUpb_MapContainer_GetOrCreateWrapper(mutval.map, field,
// self->arena);
(void)mutval;
PyErr_SetString(PyExc_NotImplementedError, "access map");
return NULL;
return PyUpb_MapContainer_GetOrCreateWrapper(mutval.map, field,
self->arena);
} else {
return PyUpb_RepeatedContainer_GetOrCreateWrapper(mutval.array, field,
self->arena);
@ -1036,7 +1026,8 @@ static PyObject* PyUpb_CMessage_Clear(PyUpb_CMessage* self, PyObject* args) {
static void PyUpb_CMessage_AbandonField(PyUpb_CMessage* self,
const upb_fielddef* f) {
if (self->unset_subobj_map && upb_fielddef_issubmsg(f)) {
if (self->unset_subobj_map && upb_fielddef_issubmsg(f) &&
!upb_fielddef_isseq(f)) {
PyObject* sub = PyUpb_WeakMap_Get(self->unset_subobj_map, f);
if (sub) {
PyUpb_CMessage_AssureWritable((PyUpb_CMessage*)sub);
@ -1089,8 +1080,7 @@ static PyObject* PyUpb_CMessage_ClearField(PyUpb_CMessage* self,
obj = PyUpb_ObjCache_Get(msgval.map_val);
}
if (obj) {
// TODO(haberman) add when maps are checked in.
// PyUpb_MapContainer_Invalidate(obj);
PyUpb_MapContainer_Invalidate(obj);
Py_DECREF(obj);
}
}

@ -59,41 +59,14 @@ message_test.Proto2Test.testMergeFromExtensions.__unittest_expecting_failure__ =
message_test.Proto2Test.testParsingMerge.__unittest_expecting_failure__ = True
message_test.Proto2Test.testPickleIncompleteProto.__unittest_expecting_failure__ = True
message_test.Proto2Test.testPythonicInit.__unittest_expecting_failure__ = True
message_test.Proto2Test.testUnknownEnumMap.__unittest_expecting_failure__ = True
message_test.Proto2Test.test_documentation.__unittest_expecting_failure__ = True
message_test.Proto3Test.testCopyFromBadType.__unittest_expecting_failure__ = True
message_test.Proto3Test.testIntegerMapWithLongs.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapAssignmentCausesPresence.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapAssignmentCausesPresenceForSubmessages.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapByteSize.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapConstruction.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapDelete.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapDeterministicSerialization.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapEntryAlwaysSerialized.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapFieldRaisesCorrectError.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapFindInitializationErrorsSmokeTest.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapGet.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapItems.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapIterInvalidatedByClearField.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapIteration.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapIterationClearMessage.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapMergeFrom.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapMessageFieldConstruction.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapScalarFieldConstruction.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapValidAfterFieldCleared.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapsAreMapping.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMapsCompare.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMergeFrom.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMergeFromBadType.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMessageMap.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMessageMapItemValidAfterTopMessageCleared.__unittest_expecting_failure__ = True
message_test.Proto3Test.testMessageMapValidAfterFieldCleared.__unittest_expecting_failure__ = True
message_test.Proto3Test.testModifyMapWhileIterating.__unittest_expecting_failure__ = True
message_test.Proto3Test.testNestedMessageMapItemDelete.__unittest_expecting_failure__ = True
message_test.Proto3Test.testScalarMap.__unittest_expecting_failure__ = True
message_test.Proto3Test.testScalarMapDefaults.__unittest_expecting_failure__ = True
message_test.Proto3Test.testStringUnicodeConversionInMap.__unittest_expecting_failure__ = True
message_test.Proto3Test.testSubmessageMap.__unittest_expecting_failure__ = True
message_test.OversizeProtosTest.testAssertOversizeProto.__unittest_expecting_failure__ = True
message_test.OversizeProtosTest.testSucceedOversizeProto.__unittest_expecting_failure__ = True

@ -35,7 +35,6 @@ unknown_fields_test.UnknownFieldsAccessorsTest.testSubUnknownFields.__unittest_e
unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownExtensions.__unittest_expecting_failure__ = True
unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownField.__unittest_expecting_failure__ = True
unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownFieldsNoMemoryLeak.__unittest_expecting_failure__ = True
unknown_fields_test.UnknownFieldsTest.testDiscardUnknownFields.__unittest_expecting_failure__ = True
unknown_fields_test.UnknownFieldsTest.testEquals.__unittest_expecting_failure__ = True
unknown_fields_test.UnknownFieldsTest.testSerializeMessageSetWireFormatUnknownExtension.__unittest_expecting_failure__ = True

@ -30,6 +30,7 @@
#include "python/descriptor.h"
#include "python/descriptor_containers.h"
#include "python/descriptor_pool.h"
#include "python/map.h"
#include "python/message.h"
#include "python/repeated.h"
@ -254,8 +255,16 @@ static const char *PyUpb_GetClassName(PyType_Spec *spec) {
PyTypeObject *PyUpb_AddClass(PyObject *m, PyType_Spec *spec) {
PyObject *type = PyType_FromSpec(spec);
const char *name = PyUpb_GetClassName(spec);
return type && PyModule_AddObject(m, name, type) == 0 ? (PyTypeObject *)type
: NULL;
int result = PyModule_AddObject(m, name, type);
return type && result == 0 ? (PyTypeObject*)type : NULL;
}
PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec,
PyObject* bases) {
PyObject* type = PyType_FromSpecWithBases(spec, bases);
const char* name = PyUpb_GetClassName(spec);
int result = PyModule_AddObject(m, name, type);
return type && result == 0 ? (PyTypeObject*)type : NULL;
}
const char *PyUpb_GetStrData(PyObject *obj) {
@ -290,7 +299,7 @@ PyMODINIT_FUNC PyInit__message(void) {
state->obj_cache = PyUpb_WeakMap_New();
if (!PyUpb_InitDescriptorContainers(m) || !PyUpb_InitDescriptorPool(m) ||
!PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) ||
!PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) || !PyUpb_Map_Init(m) ||
!PyUpb_InitMessage(m) || !PyUpb_Repeated_Init(m)) {
Py_DECREF(m);
return NULL;

@ -64,6 +64,11 @@ typedef struct {
// From descriptor_pool.c
PyTypeObject *descriptor_pool_type;
// From map.c
PyTypeObject* map_iterator_type;
PyTypeObject* message_map_container_type;
PyTypeObject* scalar_map_container_type;
// From message.c
PyObject *decode_error_class;
PyObject* descriptor_string;
@ -165,6 +170,11 @@ PyTypeObject *AddObject(PyObject *m, const char *name, PyType_Spec *spec);
// Creates a Python type from `spec` and adds it to the given module `m`.
PyTypeObject *PyUpb_AddClass(PyObject *m, PyType_Spec *spec);
// Like PyUpb_AddClass(), but allows you to specify a tuple of base classes
// in `bases`.
PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec,
PyObject* bases);
// A function that implements the tp_new slot for types that we do not allow
// users to create directly. This will immediately fail with an error message.
PyObject *PyUpb_Forbidden_New(PyObject *cls, PyObject *args, PyObject *kwds);

@ -285,6 +285,7 @@ static int _upb_mapsorter_cmpstr(const void *_a, const void *_b) {
return a.size - b.size;
}
#include <stdio.h>
bool _upb_mapsorter_pushmap(_upb_mapsorter *s, upb_descriptortype_t key_type,
const upb_map *map, _upb_sortedmap *sorted) {
int map_size = _upb_map_size(map);
@ -341,6 +342,7 @@ bool _upb_mapsorter_pushmap(_upb_mapsorter *s, upb_descriptortype_t key_type,
compar = _upb_mapsorter_cmpbool;
break;
case UPB_DESCRIPTOR_TYPE_STRING:
case UPB_DESCRIPTOR_TYPE_BYTES:
compar = _upb_mapsorter_cmpstr;
break;
default:

Loading…
Cancel
Save