Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Python: basic SymbolTable support and empty accessors.

Browse Source
pull/13171/head
Joshua Haberman 13 years ago
parent 2cf930ef51
commit fe3df2c9bc
6 changed files with 268 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      Makefile
  2. 17
      lang_ext/lua/upb.c
  3. 18
      lang_ext/python/test.py
  4. 251
      lang_ext/python/upb.c
  5. 2
      upb/bytestream.h
  6. 2
      upb/handlers.h

2
Makefile
Unescape View File

@ -405,4 +405,4 @@ $(PYTHONEXT): $(LIBUPB_PIC) lang_ext/python/upb.c
$(Q) cd lang_ext/python && $(PYTHON) setup.py build --debug install --home=build/install
pythontest: $(PYTHONEXT)
cd lang_ext/python && cp test.py build/install/lib/python && $(PYTHON) ./build/install/lib/python/test.py
cd lang_ext/python && cp test.py build/install/lib/python && valgrind $(PYTHON) ./build/install/lib/python/test.py

17
lang_ext/lua/upb.c
Unescape View File

@ -734,7 +734,7 @@ static int lupb_msgdef(lua_State *L) {
// - put the env tables for all messages and arrays in the registry, keyed by
// userdata pointer (light userdata), or by a reference using luaL_ref().
// Then we can just let upb's parse stack track the stack of env tables.
// Easy but requires all messages and arraysto be in the registry, which
// Easy but requires all messages and arrays to be in the registry, which
// seems too heavyweight.
//
// - store the stack of env tables in the Lua stack. Convenient, but requires
@ -810,21 +810,6 @@ static upb_flow_t lupb_msg_stringval_r(void *a, upb_value fval, upb_value val) {
return lupb_msg_string(a, fval, val, true);
}
#if 0
static const upb_accessor lupb_accessor = {
upb_startfield_handler *appendseq; // Repeated fields only.
upb_startfield_handler *appendsubmsg; // Submsg fields (repeated or no).
upb_value_handler *set; // Scalar fields (repeated or no).
// Readers.
upb_has_reader *has;
upb_value_reader *get;
upb_seqbegin_handler *seqbegin;
upb_seqnext_handler *seqnext;
upb_seqget_handler *seqget;
};
#endif
#define STDMSG(type, size) static upb_accessor_vtbl vtbl = { \
&lupb_msg_startsubmsg, \
&upb_stdmsg_set ## type, \

18
lang_ext/python/test.py
Unescape View File

@ -49,10 +49,24 @@ class TestMessageDef(unittest.TestCase):
self.assertEqual(msgdef1.fields(), [])
fields = [upb.FieldDef(number=1, name="field1", type=upb.TYPE_INT32)]
f = upb.FieldDef(number=1, name="field1", type=upb.TYPE_INT32)
f = upb.FieldDef()
msgdef2 = upb.MessageDef(fqname="Message2", fields=fields)
self.assertEqual(set(msgdef2.fields()), set(fields))
f2 = upb.FieldDef(number=2, name="field2", type=upb.TYPE_INT64)
msgdef2.add_field(f2)
fields.append(f2)
self.assertEqual(set(msgdef2.fields()), set(fields))
class TestSymbolTable(unittest.TestCase):
def test_construction(self):
s = upb.SymbolTable()
self.assertEqual(s.defs(), []);
s.add_def(upb.MessageDef(fqname="A"))
self.assertTrue(s.lookup("A") is not None)
self.assertTrue(s.lookup("A") is s.lookup("A"))
if __name__ == '__main__':
unittest.main()

251
lang_ext/python/upb.c
Unescape View File

@ -11,6 +11,7 @@
#include <stddef.h>
#include <Python.h>
#include "upb/def.h"
#include "upb/msg.h"
static bool streql(const char *a, const char *b) { return strcmp(a, b) == 0; }
@ -24,6 +25,11 @@ int PyUpb_ErrorInt(const char *str) {
return -1;
}
#define PyUpb_CheckStatus(status) \
if (!upb_ok(status)) return PyUpb_Error((status)->str);
static upb_accessor_vtbl *PyUpb_AccessorForField(upb_fielddef *f);
/* Object cache ***************************************************************/
@ -107,12 +113,21 @@ static PyObject *PyUpb_ObjCacheGet(void *obj, PyTypeObject *type) {
/* PyUpb_Def ******************************************************************/
static PyTypeObject *PyUpb_TypeForDef(upb_def *def);
static void PyUpb_Def_dealloc(PyObject *obj) {
PyUpb_ObjWrapper *wrapper = (void*)obj;
upb_def_unref((upb_def*)wrapper->obj);
obj->ob_type->tp_free(obj);
}
PyObject *PyUpb_Def_GetOrCreate(upb_def *def) {
return def ? PyUpb_ObjCacheGet(def, PyUpb_TypeForDef(def)) : Py_None;
}
// Will need to expand once other kinds of defs are supported.
#define Check_Def(o, badret) Check_MessageDef(o, badret)
/* PyUpb_FieldDef *************************************************************/
@ -281,7 +296,7 @@ static PyObject *PyUpb_MessageDef_new(PyTypeObject *subtype,
return PyUpb_ObjCacheGet(upb_msgdef_new(), subtype);
}
static PyObject *PyUpb_MessageDef_addfields(PyObject *o, PyObject *args);
static PyObject *PyUpb_MessageDef_add_fields(PyObject *o, PyObject *args);
static int PyUpb_MessageDef_init(PyObject *self, PyObject *args, PyObject *kwds) {
if (!kwds) return 0;
@ -290,7 +305,7 @@ static int PyUpb_MessageDef_init(PyObject *self, PyObject *args, PyObject *kwds)
while (PyDict_Next(kwds, &pos, &key, &value)) {
const char *field = PyString_AsString(key);
if (streql(field, "fields")) {
PyUpb_MessageDef_addfields(self, value);
PyUpb_MessageDef_add_fields(self, value);
} else {
PyUpb_MessageDef_setattro(self, key, value);
}
@ -336,8 +351,9 @@ static PyObject *PyUpb_MessageDef_fields(PyObject *obj, PyObject *args) {
return ret;
}
static PyObject *PyUpb_MessageDef_addfields(PyObject *o, PyObject *fields) {
static PyObject *PyUpb_MessageDef_add_fields(PyObject *o, PyObject *fields) {
upb_msgdef *m = Check_MessageDef(o, NULL);
if (!PySequence_Check(fields)) return PyUpb_Error("Must be a sequence");
Py_ssize_t len = PySequence_Length(fields);
if (len > UPB_MAX_FIELDS) return PyUpb_Error("Too many fields.");
upb_fielddef *f[len];
@ -350,7 +366,16 @@ static PyObject *PyUpb_MessageDef_addfields(PyObject *o, PyObject *fields) {
return Py_None;
}
static PyObject *PyUpb_MessageDef_add_field(PyObject *o, PyObject *field) {
upb_msgdef *m = Check_MessageDef(o, NULL);
upb_fielddef *f = Check_FieldDef(field, NULL);
upb_msgdef_addfield(m, f);
return Py_None;
}
static PyMethodDef PyUpb_MessageDef_methods[] = {
{"add_field", &PyUpb_MessageDef_add_field, METH_O, "Adds a list of fields."},
{"add_fields", &PyUpb_MessageDef_add_fields, METH_O, "Adds a list of fields."},
{"fields", &PyUpb_MessageDef_fields, METH_NOARGS, "Returns list of fields."},
{NULL, NULL}
};
@ -399,6 +424,225 @@ static PyTypeObject PyUpb_MessageDefType = {
};
static PyTypeObject *PyUpb_TypeForDef(upb_def *def) {
switch(def->type) {
case UPB_DEF_MSG: return &PyUpb_MessageDefType;
default: return NULL;
}
}
/* PyUpb_SymbolTable **********************************************************/
static PyTypeObject PyUpb_SymbolTableType;
#define Check_SymbolTable(o, badret) \
(void*)(((PyUpb_ObjWrapper*)o)->obj); do { \
if(!PyObject_TypeCheck(o, &PyUpb_SymbolTableType)) { \
PyErr_SetString(PyExc_TypeError, "must be a upb.MessageDef"); \
return badret; \
} \
} while(0)
static PyObject *PyUpb_SymbolTable_new(PyTypeObject *subtype,
PyObject *args, PyObject *kwds) {
return PyUpb_ObjCacheGet(upb_symtab_new(), subtype);
}
static int PyUpb_SymbolTable_init(PyObject *self, PyObject *args, PyObject *kwds) {
return 0;
}
static void PyUpb_SymbolTable_dealloc(PyObject *obj) {
PyUpb_ObjWrapper *wrapper = (void*)obj;
upb_symtab_unref((upb_symtab*)wrapper->obj);
obj->ob_type->tp_free(obj);
}
// narg is a lua table containing a list of defs to add.
static PyObject *PyUpb_SymbolTable_add_defs(PyObject *o, PyObject *defs) {
upb_symtab *s = Check_SymbolTable(o, NULL);
if (!PySequence_Check(defs)) return PyUpb_Error("Must be a sequence");
Py_ssize_t n = PySequence_Length(defs);
// Prevent stack overflow.
if (n > 2048) return PyUpb_Error("Too many defs");
upb_def *cdefs[n];
int i = 0;
for (i = 0; i < n; i++) {
PyObject *pydef = PySequence_GetItem(defs, i);
upb_def *def = Check_MessageDef(pydef, NULL);
cdefs[i++] = def;
upb_msgdef *md = upb_dyncast_msgdef(def);
if (!md) continue;
upb_msg_iter j;
for(j = upb_msg_begin(md); !upb_msg_done(j); j = upb_msg_next(md, j)) {
upb_fielddef *f = upb_msg_iter_field(j);
upb_fielddef_setaccessor(f, PyUpb_AccessorForField(f));
}
upb_msgdef_layout(md);
}
upb_status status = UPB_STATUS_INIT;
upb_symtab_add(s, cdefs, n, &status);
PyUpb_CheckStatus(&status);
return Py_None;
}
static PyObject *PyUpb_SymbolTable_add_def(PyObject *o, PyObject *def) {
PyObject *defs = PyList_New(1);
PyList_SetItem(defs, 0, def);
return PyUpb_SymbolTable_add_defs(o, defs);
}
// TODO: update to allow user to choose type of defs.
static PyObject *PyUpb_SymbolTable_defs(PyObject *o, PyObject *none) {
upb_symtab *s = Check_SymbolTable(o, NULL);
int count;
upb_def **defs = upb_symtab_getdefs(s, &count, UPB_DEF_ANY);
PyObject *ret = PyList_New(count);
int i;
for(i = 0; i < count; i++)
PyList_SetItem(ret, i, PyUpb_Def_GetOrCreate(defs[i]));
return ret;
}
static PyObject *PyUpb_SymbolTable_lookup(PyObject *o, PyObject *arg) {
upb_symtab *s = Check_SymbolTable(o, NULL);
const char *name = PyString_AsString(arg);
upb_def *def = upb_symtab_lookup(s, name);
return PyUpb_Def_GetOrCreate(def);
}
static PyMethodDef PyUpb_SymbolTable_methods[] = {
{"add_def", &PyUpb_SymbolTable_add_def, METH_O, NULL},
{"add_defs", &PyUpb_SymbolTable_add_defs, METH_O, NULL},
{"defs", &PyUpb_SymbolTable_defs, METH_NOARGS, NULL},
{"lookup", &PyUpb_SymbolTable_lookup, METH_O, NULL},
{NULL, NULL}
};
static PyTypeObject PyUpb_SymbolTableType = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
"upb.SymbolTable", /* tp_name */
sizeof(PyUpb_ObjWrapper), /* tp_basicsize */
0, /* tp_itemsize */
&PyUpb_SymbolTable_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* TODO */ /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
offsetof(PyUpb_ObjWrapper, weakreflist),/* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
PyUpb_SymbolTable_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
&PyUpb_SymbolTable_init, /* tp_init */
0, /* tp_alloc */
&PyUpb_SymbolTable_new, /* tp_new */
0, /* tp_free */
};
/* Accessor and PyUpb_Message *************************************************/
static upb_sflow_t PyUpb_Message_StartSequence(void *m, upb_value fval) {
upb_fielddef *f = upb_value_getfielddef(fval);
(void)f;
return UPB_CONTINUE_WITH(NULL);
}
static upb_sflow_t PyUpb_Message_StartSubmessage(void *m, upb_value fval) {
upb_fielddef *f = upb_value_getfielddef(fval);
(void)f;
return UPB_CONTINUE_WITH(NULL);
}
static upb_sflow_t PyUpb_Message_StartRepeatedSubmessage(void *a, upb_value fval) {
upb_fielddef *f = upb_value_getfielddef(fval);
(void)f;
return UPB_CONTINUE_WITH(NULL);
}
static upb_flow_t PyUpb_Message_StringValue(void *m, upb_value fval, upb_value val) {
upb_fielddef *f = upb_value_getfielddef(fval);
(void)f;
return UPB_CONTINUE;
}
static upb_flow_t PyUpb_Message_AppendStringValue(void *a, upb_value fval, upb_value val) {
upb_fielddef *f = upb_value_getfielddef(fval);
(void)f;
return UPB_CONTINUE;
}
#define STDMSG(type, size) static upb_accessor_vtbl vtbl = { \
&PyUpb_Message_StartSubmessage, \
&upb_stdmsg_set ## type, \
&PyUpb_Message_StartSequence, \
&PyUpb_Message_StartRepeatedSubmessage, \
&upb_stdmsg_set ## type ## _r, \
&upb_stdmsg_has, \
&upb_stdmsg_getptr, \
&upb_stdmsg_get ## type, \
&upb_stdmsg_seqbegin, \
&upb_stdmsg_ ## size ## byte_seqnext, \
&upb_stdmsg_seqget ## type};
#define RETURN_STDMSG(type, size) { STDMSG(type, size); return &vtbl; }
static upb_accessor_vtbl *PyUpb_AccessorForField(upb_fielddef *f) {
switch (f->type) {
case UPB_TYPE(DOUBLE): RETURN_STDMSG(double, 8)
case UPB_TYPE(FLOAT): RETURN_STDMSG(float, 4)
case UPB_TYPE(UINT64):
case UPB_TYPE(FIXED64): RETURN_STDMSG(uint64, 8)
case UPB_TYPE(INT64):
case UPB_TYPE(SFIXED64):
case UPB_TYPE(SINT64): RETURN_STDMSG(int64, 8)
case UPB_TYPE(INT32):
case UPB_TYPE(SINT32):
case UPB_TYPE(ENUM):
case UPB_TYPE(SFIXED32): RETURN_STDMSG(int32, 4)
case UPB_TYPE(UINT32):
case UPB_TYPE(FIXED32): RETURN_STDMSG(uint32, 4)
case UPB_TYPE(BOOL): { STDMSG(bool, 1); return &vtbl; }
case UPB_TYPE(GROUP):
case UPB_TYPE(MESSAGE): RETURN_STDMSG(ptr, 8) // TODO: 32-bit
case UPB_TYPE(STRING):
case UPB_TYPE(BYTES): {
STDMSG(ptr, 8);
vtbl.set = &PyUpb_Message_StringValue;
vtbl.append = &PyUpb_Message_AppendStringValue;
return &vtbl;
}
}
return NULL;
}
/* Toplevel *******************************************************************/
static PyMethodDef methods[] = {
@ -417,6 +661,7 @@ PyMODINIT_FUNC initupb(void) {
PyUpb_AddType(mod, "FieldDef", &PyUpb_FieldDefType);
PyUpb_AddType(mod, "MessageDef", &PyUpb_MessageDefType);
PyUpb_AddType(mod, "SymbolTable", &PyUpb_SymbolTableType);
PyModule_AddIntConstant(mod, "LABEL_OPTIONAL", UPB_LABEL(OPTIONAL));
PyModule_AddIntConstant(mod, "LABEL_REQUIRED", UPB_LABEL(REQUIRED));

2
upb/bytestream.h
Unescape View File

@ -278,7 +278,7 @@ void upb_stringsink_reset(upb_stringsink *s, char *str, size_t size);
const char *upb_stringsink_release(upb_stringsink *s, size_t *len);
// Returns the upb_bytesink* for this stringsrc. Invalidated by reset above.
upb_bytesink *upb_stringsink_bytesink();
upb_bytesink *upb_stringsink_bytesink(void);
#ifdef __cplusplus
} /* extern "C" */

2
upb/handlers.h
Unescape View File

@ -230,7 +230,7 @@ struct _upb_handlers {
};
typedef struct _upb_handlers upb_handlers;
upb_handlers *upb_handlers_new();
upb_handlers *upb_handlers_new(void);
void upb_handlers_ref(upb_handlers *h);
void upb_handlers_unref(upb_handlers *h);

Write Preview
Loading…
Cancel
Save