Initial commit of ExtensionDict.

pull/13171/head
Joshua Haberman 3 years ago
parent 3132ac8d84
commit 627c44be1f
  1. 2
      python/BUILD
  2. 190
      python/extension_dict.c
  3. 39
      python/extension_dict.h
  4. 8
      python/message.c
  5. 1
      python/pb_unit_tests/generator_test_wrapper.py
  6. 1
      python/pb_unit_tests/json_format_test_wrapper.py
  7. 3
      python/pb_unit_tests/keywords_test_wrapper.py
  8. 1
      python/pb_unit_tests/message_test_wrapper.py
  9. 12
      python/pb_unit_tests/reflection_test_wrapper.py
  10. 4
      python/pb_unit_tests/text_format_test_wrapper.py
  11. 4
      python/protobuf.c
  12. 4
      python/protobuf.h
  13. 6
      upb/msg.c
  14. 5
      upb/msg.h

@ -39,6 +39,8 @@ cc_binary(
"descriptor_containers.h",
"descriptor_pool.c",
"descriptor_pool.h",
"extension_dict.c",
"extension_dict.h",
"map.c",
"map.h",
"message.c",

@ -0,0 +1,190 @@
/*
* Copyright (c) 2009-2021, Google LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Google LLC nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "python/extension_dict.h"
#include "python/message.h"
#include "python/protobuf.h"
// -----------------------------------------------------------------------------
// ExtensionDict
// -----------------------------------------------------------------------------
typedef struct {
PyObject_HEAD
PyObject* msg; // Owning ref to our parent pessage.
} PyUpb_ExtensionDict;
PyObject* PyUpb_ExtensionDict_New(PyObject* msg) {
PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
PyUpb_ExtensionDict* ext_dict =
(void*)PyType_GenericAlloc(state->extension_dict_type, 0);
ext_dict->msg = msg;
Py_INCREF(ext_dict->msg);
return &ext_dict->ob_base;
}
static PyObject* PyUpb_ExtensionDict_FindExtensionByName(PyObject* _self,
PyObject* key) {
PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
const char* name = PyUpb_GetStrData(key);
const upb_msgdef* m = PyUpb_CMessage_GetMsgdef(self->msg);
const upb_filedef* file = upb_msgdef_file(m);
const upb_symtab* symtab = upb_filedef_symtab(file);
const upb_fielddef* ext = upb_symtab_lookupext(symtab, name);
if (ext) {
return PyUpb_FieldDescriptor_Get(ext);
} else {
Py_RETURN_NONE;
}
}
static PyObject* PyUpb_ExtensionDict_FindExtensionByNumber(PyObject* _self,
PyObject* arg) {
PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
const upb_msgdef* m = PyUpb_CMessage_GetMsgdef(self->msg);
const upb_msglayout* l = upb_msgdef_layout(m);
const upb_filedef* file = upb_msgdef_file(m);
const upb_symtab* symtab = upb_filedef_symtab(file);
const upb_extreg* reg = upb_symtab_extreg(symtab);
int64_t number = PyLong_AsLong(arg);
const upb_msglayout_ext* ext =
(upb_msglayout_ext*)_upb_extreg_get(reg, l, number);
if (ext) {
const upb_fielddef* f = _upb_symtab_lookupextfield(symtab, ext);
return PyUpb_FieldDescriptor_Get(f);
} else {
Py_RETURN_NONE;
}
}
static void PyUpb_ExtensionDict_Dealloc(PyUpb_ExtensionDict* self) {
PyUpb_CMessage_ClearExtensionDict(self->msg);
Py_DECREF(self->msg);
PyUpb_Dealloc(self);
}
static Py_ssize_t PyUpb_ExtensionDict_Length(PyObject* _self) {
PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
upb_msg* msg = PyUpb_CMessage_GetIfReified(self->msg);
return msg ? upb_msg_extcount(msg) : 0;
}
static PyObject* PyUpb_ExtensionDict_Subscript(PyObject* _self, PyObject* key) {
PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
const upb_fielddef* f = PyUpb_FieldDescriptor_GetDef(key);
if (!f) return NULL;
if (!upb_fielddef_isextension(f)) {
return PyErr_Format(PyExc_KeyError, "Field %s is not an extension\n",
upb_fielddef_fullname(f));
}
return PyUpb_CMessage_GetFieldValue(self->msg, f);
}
int PyUpb_ExtensionDict_AssignSubscript(PyObject* _self, PyObject* key,
PyObject* val) {
PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
const upb_fielddef* f = PyUpb_FieldDescriptor_GetDef(key);
if (!f) return -1;
if (!upb_fielddef_isextension(f)) {
PyErr_Format(PyExc_KeyError, "Field %s is not an extension\n",
upb_fielddef_fullname(f));
return -1;
}
return PyUpb_CMessage_SetFieldValue(self->msg, f, val);
}
static PyMethodDef PyUpb_ExtensionDict_Methods[] = {
{"_FindExtensionByName", PyUpb_ExtensionDict_FindExtensionByName, METH_O,
"Finds an extension by name."},
{"_FindExtensionByNumber", PyUpb_ExtensionDict_FindExtensionByNumber,
METH_O, "Finds an extension by number."},
{NULL, NULL},
};
static PyType_Slot PyUpb_ExtensionDict_Slots[] = {
{Py_tp_dealloc, PyUpb_ExtensionDict_Dealloc},
{Py_tp_methods, PyUpb_ExtensionDict_Methods},
//{Py_tp_getset, PyUpb_ExtensionDict_Getters},
//{Py_tp_hash, PyObject_HashNotImplemented},
//{Py_tp_richcompare, PyUpb_ExtensionDict_RichCompare},
//{Py_tp_iter, PyUpb_ExtensionDict_GetIter},
{Py_sq_length, PyUpb_ExtensionDict_Length},
//{Py_sq_contains, PyUpb_ExtensionDict_Contains},
{Py_mp_length, PyUpb_ExtensionDict_Length},
{Py_mp_subscript, PyUpb_ExtensionDict_Subscript},
{Py_mp_ass_subscript, PyUpb_ExtensionDict_AssignSubscript},
{0, NULL}};
static PyType_Spec PyUpb_ExtensionDict_Spec = {
PYUPB_MODULE_NAME ".ExtensionDict", // tp_name
sizeof(PyUpb_ExtensionDict), // tp_basicsize
0, // tp_itemsize
Py_TPFLAGS_DEFAULT, // tp_flags
PyUpb_ExtensionDict_Slots,
};
// -----------------------------------------------------------------------------
// ExtensionIterator
// -----------------------------------------------------------------------------
typedef struct {
PyObject_HEAD
PyObject* arena;
} PyUpb_ExtensionIterator;
static void PyUpb_ExtensionIterator_Dealloc(PyUpb_ExtensionDict* self) {
PyUpb_Dealloc(self);
}
static PyType_Slot PyUpb_ExtensionIterator_Slots[] = {
{Py_tp_dealloc, PyUpb_ExtensionIterator_Dealloc},
{Py_tp_iter, PyObject_SelfIter},
//{Py_tp_iter, PyUpb_ExtensionIterator_GetIter},
//{Py_tp_iternext, PyUpb_ExtensionIterator_IterNext},
{0, NULL}};
static PyType_Spec PyUpb_ExtensionIterator_Spec = {
PYUPB_MODULE_NAME ".ExtensionIterator", // tp_name
sizeof(PyUpb_ExtensionDict), // tp_basicsize
0, // tp_itemsize
Py_TPFLAGS_DEFAULT, // tp_flags
PyUpb_ExtensionIterator_Slots,
};
// -----------------------------------------------------------------------------
// Top Level
// -----------------------------------------------------------------------------
bool PyUpb_InitExtensionDict(PyObject* m) {
PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m);
s->extension_dict_type = PyUpb_AddClass(m, &PyUpb_ExtensionDict_Spec);
s->extension_iterator_type = PyUpb_AddClass(m, &PyUpb_ExtensionIterator_Spec);
return s->extension_dict_type && s->extension_iterator_type;
}

@ -0,0 +1,39 @@
/*
* Copyright (c) 2009-2021, Google LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Google LLC nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef PYUPB_EXTENSION_DICT_H__
#define PYUPB_EXTENSION_DICT_H__
#include <stdbool.h>
#include "python/python.h"
PyObject* PyUpb_ExtensionDict_New(PyObject* msg);
bool PyUpb_InitExtensionDict(PyObject* m);
#endif // PYUPB_EXTENSION_DICT_H__

@ -29,6 +29,7 @@
#include "python/convert.h"
#include "python/descriptor.h"
#include "python/extension_dict.h"
#include "python/map.h"
#include "python/repeated.h"
#include "upb/def.h"
@ -1241,11 +1242,8 @@ static PyObject* PyUpb_CMessage_GetExtensionDict(PyObject* _self,
return NULL;
}
// TODO(haberman): re-enable when ExtensionDict is checked in.
// self->ext_dict = PyUpb_ExtensionDict_New(_self);
// return self->ext_dict;
PyErr_SetString(PyExc_NotImplementedError, "get extension dict");
return NULL;
self->ext_dict = PyUpb_ExtensionDict_New(_self);
return self->ext_dict;
}
static PyGetSetDef PyUpb_CMessage_Getters[] = {

@ -26,7 +26,6 @@
from google.protobuf.internal import generator_test
import unittest
generator_test.GeneratorTest.testBadIdentifiers.__unittest_expecting_failure__ = True
generator_test.GeneratorTest.testExtensionScope.__unittest_expecting_failure__ = True
generator_test.GeneratorTest.testMessageWithCustomOptions.__unittest_expecting_failure__ = True
generator_test.GeneratorTest.testOneof.__unittest_expecting_failure__ = True

@ -32,7 +32,6 @@ json_format_test.JsonFormatTest.testAnyMessage.__unittest_expecting_failure__ =
json_format_test.JsonFormatTest.testDurationMessage.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testEmptyMessageToJson.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testExtensionSerializationDictMatchesProto3Spec.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testExtensionSerializationDictMatchesProto3SpecMore.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testExtensionSerializationJsonMatchesProto3Spec.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testExtensionToDictAndBack.__unittest_expecting_failure__ = True
json_format_test.JsonFormatTest.testExtensionToDictAndBackWithScalar.__unittest_expecting_failure__ = True

@ -26,8 +26,5 @@
from google.protobuf.internal import keywords_test
import unittest
keywords_test.KeywordsConflictTest.testExtension.__unittest_expecting_failure__ = True
keywords_test.KeywordsConflictTest.testExtensionForNestedMessage.__unittest_expecting_failure__ = True
if __name__ == '__main__':
unittest.main(module=keywords_test, verbosity=2)

@ -55,7 +55,6 @@ message_test.MessageTest.testRepeatedContains_proto3.__unittest_expecting_failur
message_test.Proto2Test.testExtensionsErrors.__unittest_expecting_failure__ = True
message_test.Proto2Test.testGoldenExtensions.__unittest_expecting_failure__ = True
message_test.Proto2Test.testGoldenPackedExtensions.__unittest_expecting_failure__ = True
message_test.Proto2Test.testMergeFromExtensions.__unittest_expecting_failure__ = True
message_test.Proto2Test.testParsingMerge.__unittest_expecting_failure__ = True
message_test.Proto2Test.testPickleIncompleteProto.__unittest_expecting_failure__ = True
message_test.Proto2Test.testPythonicInit.__unittest_expecting_failure__ = True

@ -26,12 +26,6 @@
from google.protobuf.internal import reflection_test
import unittest
reflection_test.ByteSizeTest.testCacheInvalidationForNonrepeatedMessage.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testCacheInvalidationForNonrepeatedScalar.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testCacheInvalidationForRepeatedMessage.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testCacheInvalidationForRepeatedScalar.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testExtensions.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testPackedExtensions.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testRepeatedCompositesDelete.__unittest_expecting_failure__ = True
reflection_test.ByteSizeTest.testRepeatedScalarsRemove.__unittest_expecting_failure__ = True
reflection_test.ClassAPITest.testMakeClassWithNestedDescriptor.__unittest_expecting_failure__ = True
@ -40,17 +34,12 @@ reflection_test.MutualRecursionEqualityTest.testEqualityWithMutualRecursion.__un
reflection_test.OptionsTest.testMessageOptions.__unittest_expecting_failure__ = True
reflection_test.OptionsTest.testPackedOptions.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testClear.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testDisconnectionAfterClearingEmptyMessage.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testExtensionContainsError.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testExtensionDelete.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testExtensionFailureModes.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testExtensionIter.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testHasBitsForAncestorsOfExtendedMessage.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testIsInitialized.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testListFieldsAndExtensions.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testMergeFromExtensionsNestedMessage.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testMergeFromExtensionsRepeated.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testMergeFromExtensionsSingular.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testNestedExtensions.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testRepeatedCompositeConstructor.__unittest_expecting_failure__ = True
reflection_test.Proto2ReflectionTest.testRepeatedCompositeRemove.__unittest_expecting_failure__ = True
@ -108,7 +97,6 @@ reflection_test.SerializationTest.testInitRepeatedKwargs.__unittest_expecting_fa
reflection_test.SerializationTest.testInitRequiredForeignKwargs.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testInitRequiredKwargs.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testMessageSetWireFormat.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testMessageSetWireFormatUnknownExtension.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testParsePackedFromUnpacked.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testParseUnpackedFromPacked.__unittest_expecting_failure__ = True
reflection_test.SerializationTest.testSerializeAllExtensions.__unittest_expecting_failure__ = True

@ -41,12 +41,8 @@ text_format_test.OnlyWorksWithProto2RightNowTests.testPrintMap.__unittest_expect
text_format_test.OnlyWorksWithProto2RightNowTests.testPrintMapUsingCppImplementation.__unittest_expecting_failure__ = True
text_format_test.OnlyWorksWithProto2RightNowTests.testPrintUnknownFields.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testExtensionInsideAnyMessage.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testMergeDuplicateExtensionScalars.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseAllExtensions.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseAllowedUnknownExtension.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseBadExtension.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseDuplicateExtensionMessages.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseDuplicateExtensionScalars.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseGoldenExtensions.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseMap.__unittest_expecting_failure__ = True
text_format_test.Proto2Tests.testParseMessageByFieldNumber.__unittest_expecting_failure__ = True

@ -30,6 +30,7 @@
#include "python/descriptor.h"
#include "python/descriptor_containers.h"
#include "python/descriptor_pool.h"
#include "python/extension_dict.h"
#include "python/map.h"
#include "python/message.h"
#include "python/repeated.h"
@ -305,7 +306,8 @@ PyMODINIT_FUNC PyInit__message(void) {
state->obj_cache = PyUpb_WeakMap_New();
if (!PyUpb_InitDescriptorContainers(m) || !PyUpb_InitDescriptorPool(m) ||
!PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) || !PyUpb_Map_Init(m) ||
!PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) ||
!PyUpb_InitExtensionDict(m) || !PyUpb_Map_Init(m) ||
!PyUpb_InitMessage(m) || !PyUpb_Repeated_Init(m)) {
Py_DECREF(m);
return NULL;

@ -64,6 +64,10 @@ typedef struct {
// From descriptor_pool.c
PyTypeObject *descriptor_pool_type;
// From extension_dict.c
PyTypeObject* extension_dict_type;
PyTypeObject* extension_iterator_type;
// From map.c
PyTypeObject* map_iterator_type;
PyTypeObject* message_map_container_type;

@ -162,6 +162,12 @@ upb_msg_ext *_upb_msg_getorcreateext(upb_msg *msg, const upb_msglayout_ext *e,
return ext;
}
size_t upb_msg_extcount(const upb_msg *msg) {
size_t count;
_upb_msg_getexts(msg, &count);
return count;
}
/** upb_array *****************************************************************/
bool _upb_array_realloc(upb_array *arr, size_t min_size, upb_arena *arena) {

@ -44,6 +44,8 @@
extern "C" {
#endif
/** upb_msg *******************************************************************/
typedef void upb_msg;
/* For users these are opaque. They can be obtained from upb_msgdef_layout()
@ -59,6 +61,9 @@ void upb_msg_addunknown(upb_msg *msg, const char *data, size_t len,
/* Returns a reference to the message's unknown data. */
const char *upb_msg_getunknown(const upb_msg *msg, size_t *len);
/* Returns the number of extensions present in this message. */
size_t upb_msg_extcount(const upb_msg *msg);
/** upb_extreg *******************************************************************/
/* Extension registry: a dynamic data structure that stores a map of:

Loading…
Cancel
Save