Chiebot-Mirror
/
protobuf
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Maps encode and decode successfully!

Browse Source
pull/13171/head
Joshua Haberman 5 years ago
parent 4c57b1fefd
commit 382f92a87f
10 changed files with 155 additions and 317 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 334
      tests/bindings/lua/test_upb.lua
  2. 68
      upb/decode.c
  3. 7
      upb/def.c
  4. 9
      upb/encode.c
  5. 17
      upb/msg.c
  6. 17
      upb/msg.h
  7. 1
      upb/reflection.c
  8. 6
      upb/reflection.h
  9. 2
      upb/upb.h
  10. 11
      upbc/message_layout.cc

334
tests/bindings/lua/test_upb.lua
Unescape View File

@ -42,285 +42,6 @@ function test_def_readers()
assert_equal(2, e:value("BAZ"))
end
--[[
function test_enumdef()
function test_iteration()
-- Test that we cannot crash the process even if we modify the set of fields
-- during iteration.
local md = upb.MessageDef{full_name = "TestMessage"}
for i=1,10 do
md:add(upb.FieldDef{
name = "field" .. tostring(i),
number = 1000 - i,
type = upb.TYPE_INT32
})
end
local add = #md
for f in md:fields() do
if add > 0 then
add = add - 1
for i=10000,11000 do
local field_name = "field" .. tostring(i)
-- We want to add fields to the table to trigger a table resize,
-- but we must skip it if the field name or number already exists
-- otherwise it will raise an error.
if md:field(field_name) == nil and
md:field(i) == nil then
md:add(upb.FieldDef{
name = field_name,
number = i,
type = upb.TYPE_INT32
})
end
end
end
end
-- Test that iterators don't crash the process even if the MessageDef goes
-- out of scope.
--
-- Note: have previously verified that this can indeed crash the process if
-- we do not explicitly add a reference from the iterator to the underlying
-- MessageDef.
local iter = md:fields()
md = nil
collectgarbage()
while iter() do
end
local ed = upb.EnumDef{
values = {
{"FOO", 1},
{"BAR", 77},
}
}
iter = ed:values()
ed = nil
collectgarbage()
while iter() do
end
end
function test_msgdef_setters()
local md = upb.MessageDef()
md:set_full_name("Message1")
assert_equal("Message1", md:full_name())
local f = upb.FieldDef{name = "field1", number = 3, type = upb.TYPE_DOUBLE}
md:add(f)
assert_equal(1, #md)
assert_equal(f, md:field("field1"))
end
function test_msgdef_errors()
assert_error(function() upb.MessageDef{bad_initializer_key = 5} end)
local md = upb.MessageDef()
assert_error(function()
-- Duplicate field number.
upb.MessageDef{
fields = {
upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32},
upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32}
}
}
end)
assert_error(function()
-- Duplicate field name.
upb.MessageDef{
fields = {
upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32},
upb.FieldDef{name = "field1", number = 2, type = upb.TYPE_INT32}
}
}
end)
assert_error(function()
-- Duplicate field name.
upb.MessageDef{
fields = {
upb.OneofDef{name = "field1", fields = {
upb.FieldDef{name = "field2", number = 1, type = upb.TYPE_INT32},
}},
upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_INT32}
}
}
end)
-- attempt to set a name with embedded NULLs.
assert_error_match("names cannot have embedded NULLs", function()
md:set_full_name("abc\0def")
end)
upb.freeze(md)
-- Attempt to mutate frozen MessageDef.
assert_error_match("frozen", function()
md:add(upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32})
end)
assert_error_match("frozen", function()
md:set_full_name("abc")
end)
-- Attempt to freeze a msgdef without freezing its subdef.
assert_error_match("is not frozen or being frozen", function()
m1 = upb.MessageDef()
upb.freeze(
upb.MessageDef{
fields = {
upb.FieldDef{name = "f1", number = 1, type = upb.TYPE_MESSAGE,
subdef = m1}
}
}
)
end)
end
function test_symtab()
local empty = upb.SymbolTable()
assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ANY)))
assert_equal(0, #iter_to_array(empty:defs(upb.DEF_MSG)))
assert_equal(0, #iter_to_array(empty:defs(upb.DEF_ENUM)))
local symtab = upb.SymbolTable{
upb.MessageDef{full_name = "TestMessage"},
upb.MessageDef{full_name = "ContainingMessage", fields = {
upb.FieldDef{name = "field1", number = 1, type = upb.TYPE_INT32},
upb.FieldDef{name = "field2", number = 2, type = upb.TYPE_MESSAGE,
subdef_name = ".TestMessage"}
}
}
}
local msgdef1 = symtab:lookup("TestMessage")
local msgdef2 = symtab:lookup("ContainingMessage")
assert_not_nil(msgdef1)
assert_not_nil(msgdef2)
assert_equal(msgdef1, msgdef2:field("field2"):subdef())
assert_true(msgdef1:is_frozen())
assert_true(msgdef2:is_frozen())
symtab:add{
upb.MessageDef{full_name = "ContainingMessage2", fields = {
upb.FieldDef{name = "field5", number = 5, type = upb.TYPE_MESSAGE,
subdef = msgdef2}
}
}
}
local msgdef3 = symtab:lookup("ContainingMessage2")
assert_not_nil(msgdef3)
assert_equal(msgdef3:field("field5"):subdef(), msgdef2)
end
function test_msg_primitives()
local function test_for_numeric_type(upb_type, val, too_big, too_small, bad3)
local symtab = upb.SymbolTable{
upb.MessageDef{full_name = "TestMessage", fields = {
upb.FieldDef{name = "f", number = 1, type = upb_type},
}
}
}
factory = upb.MessageFactory(symtab)
TestMessage = factory:get_message_class("TestMessage")
msg = TestMessage()
-- Defaults to zero
assert_equal(0, msg.f)
msg.f = 0
assert_equal(0, msg.f)
msg.f = val
assert_equal(val, msg.f)
local errmsg = "not an integer or out of range"
if too_small then
assert_error_match(errmsg, function() msg.f = too_small end)
end
if too_big then
assert_error_match(errmsg, function() msg.f = too_big end)
end
if bad3 then
assert_error_match(errmsg, function() msg.f = bad3 end)
end
-- Can't assign other Lua types.
errmsg = "bad argument #3"
assert_error_match(errmsg, function() msg.f = "abc" end)
assert_error_match(errmsg, function() msg.f = true end)
assert_error_match(errmsg, function() msg.f = false end)
assert_error_match(errmsg, function() msg.f = nil end)
assert_error_match(errmsg, function() msg.f = {} end)
assert_error_match(errmsg, function() msg.f = print end)
assert_error_match(errmsg, function() msg.f = array end)
end
local symtab = upb.SymbolTable{
upb.MessageDef{full_name = "TestMessage", fields = {
upb.FieldDef{
name = "i32", number = 1, type = upb.TYPE_INT32, default = 1},
upb.FieldDef{
name = "u32", number = 2, type = upb.TYPE_UINT32, default = 2},
upb.FieldDef{
name = "i64", number = 3, type = upb.TYPE_INT64, default = 3},
upb.FieldDef{
name = "u64", number = 4, type = upb.TYPE_UINT64, default = 4},
upb.FieldDef{
name = "dbl", number = 5, type = upb.TYPE_DOUBLE, default = 5},
upb.FieldDef{
name = "flt", number = 6, type = upb.TYPE_FLOAT, default = 6},
upb.FieldDef{
name = "bool", number = 7, type = upb.TYPE_BOOL, default = true},
}
}
}
factory = upb.MessageFactory(symtab)
TestMessage = factory:get_message_class("TestMessage")
msg = TestMessage()
-- Unset member returns default value.
-- TODO(haberman): re-enable these when we have descriptor-based reflection.
-- assert_equal(1, msg.i32)
-- assert_equal(2, msg.u32)
-- assert_equal(3, msg.i64)
-- assert_equal(4, msg.u64)
-- assert_equal(5, msg.dbl)
-- assert_equal(6, msg.flt)
-- assert_equal(true, msg.bool)
-- Attempts to access non-existent fields fail.
assert_error_match("no such field", function() msg.no_such = 1 end)
msg.i32 = 10
msg.u32 = 20
msg.i64 = 30
msg.u64 = 40
msg.dbl = 50
msg.flt = 60
msg.bool = true
assert_equal(10, msg.i32)
assert_equal(20, msg.u32)
assert_equal(30, msg.i64)
assert_equal(40, msg.u64)
assert_equal(50, msg.dbl)
assert_equal(60, msg.flt)
assert_equal(true, msg.bool)
test_for_numeric_type(upb.TYPE_UINT32, 2^32 - 1, 2^32, -1, 5.1)
test_for_numeric_type(upb.TYPE_UINT64, 2^62, 2^64, -1, bad64)
test_for_numeric_type(upb.TYPE_INT32, 2^31 - 1, 2^31, -2^31 - 1, 5.1)
test_for_numeric_type(upb.TYPE_INT64, 2^61, 2^63, -2^64, bad64)
test_for_numeric_type(upb.TYPE_FLOAT, 2^20)
test_for_numeric_type(upb.TYPE_DOUBLE, 10^101)
end
==]]
function test_msg_map()
msg = test_messages_proto3.TestAllTypesProto3()
msg.map_int32_int32[5] = 10
@ -335,6 +56,21 @@ function test_msg_map()
assert_equal(12, msg2.map_int32_int32[6])
end
function test_msg_string_map()
msg = test_messages_proto3.TestAllTypesProto3()
msg.map_string_string["foo"] = "bar"
msg.map_string_string["baz"] = "quux"
assert_nil(msg.map_string_string["abc"])
assert_equal("bar", msg.map_string_string["foo"])
assert_equal("quux", msg.map_string_string["baz"])
local serialized = upb.encode(msg)
assert_true(#serialized > 0)
local msg2 = upb.decode(test_messages_proto3.TestAllTypesProto3, serialized)
assert_equal("bar", msg2.map_string_string["foo"])
assert_equal("quux", msg2.map_string_string["baz"])
end
function test_msg_array()
msg = test_messages_proto3.TestAllTypesProto3()
@ -472,6 +208,46 @@ local numeric_types = {
},
}
function test_msg_primitives()
local msg = test_messages_proto3.TestAllTypesProto3{
optional_int32 = 10,
optional_uint32 = 20,
optional_int64 = 30,
optional_uint64 = 40,
optional_double = 50,
optional_float = 60,
optional_sint32 = 70,
optional_sint64 = 80,
optional_fixed32 = 90,
optional_fixed64 = 100,
optional_sfixed32 = 110,
optional_sfixed64 = 120,
optional_bool = true,
optional_string = "abc",
optional_nested_message = test_messages_proto3['TestAllTypesProto3.NestedMessage']{a = 123},
}
-- Attempts to access non-existent fields fail.
assert_error_match("no such field", function() msg.no_such = 1 end)
assert_equal(10, msg.optional_int32)
assert_equal(20, msg.optional_uint32)
assert_equal(30, msg.optional_int64)
assert_equal(40, msg.optional_uint64)
assert_equal(50, msg.optional_double)
assert_equal(60, msg.optional_float)
assert_equal(70, msg.optional_sint32)
assert_equal(80, msg.optional_sint64)
assert_equal(90, msg.optional_fixed32)
assert_equal(100, msg.optional_fixed64)
assert_equal(110, msg.optional_sfixed32)
assert_equal(120, msg.optional_sfixed64)
assert_equal(true, msg.optional_bool)
assert_equal("abc", msg.optional_string)
assert_equal(123, msg.optional_nested_message.a)
end
function test_string_array()
local function test_for_string_type(upb_type)
local array = upb.Array(upb_type)

68
upb/decode.c
Unescape View File

@ -6,7 +6,7 @@
#include "upb/port_def.inc"
/* Maps descriptor type -> upb field type. */
const uint8_t upb_desctype_to_fieldtype[] = {
const uint8_t desctype_to_fieldtype[] = {
UPB_WIRE_TYPE_END_GROUP, /* ENDGROUP */
UPB_TYPE_DOUBLE, /* DOUBLE */
UPB_TYPE_FLOAT, /* FLOAT */
@ -198,7 +198,7 @@ static upb_array *upb_getorcreatearr(upb_decframe *frame,
upb_array *arr = upb_getarr(frame, field);
if (!arr) {
upb_fieldtype_t type = upb_desctype_to_fieldtype[field->descriptortype];
upb_fieldtype_t type = desctype_to_fieldtype[field->descriptortype];
arr = upb_array_new(frame->state->arena, type);
CHK(arr);
*(upb_array**)&frame->msg[field->offset] = arr;
@ -463,6 +463,63 @@ static bool upb_decode_toarray(upb_decstate *d, upb_decframe *frame,
UPB_UNREACHABLE();
}
static bool upb_decode_mapfield(upb_decstate *d, upb_decframe *frame,
const upb_msglayout_field *field, int len) {
/* Max map entry size is string key/val. */
size_t size = sizeof(upb_msg_internal) + (sizeof(upb_strview) * 2);
char submsg[size];
char *submsg_ptr = &submsg[sizeof(upb_msg_internal)];
upb_map *map = *(upb_map**)&frame->msg[field->offset];
upb_alloc *alloc = upb_arena_alloc(d->arena);
const upb_msglayout *entry = frame->layout->submsgs[field->submsg_index];
upb_value val;
const char *key;
size_t key_size;
if (!map) {
/* Lazily create map. */
const upb_msglayout_field *key_field = &entry->fields[0];
const upb_msglayout_field *val_field = &entry->fields[1];
upb_fieldtype_t key_type = desctype_to_fieldtype[key_field->descriptortype];
upb_fieldtype_t val_type = desctype_to_fieldtype[val_field->descriptortype];
UPB_ASSERT(key_field->number == 1);
UPB_ASSERT(val_field->number == 2);
UPB_ASSERT(key_field->offset == 0);
UPB_ASSERT(val_field->offset == sizeof(upb_strview));
map = upb_map_new(frame->state->arena, key_type, val_type);
*(upb_map**)&frame->msg[field->offset] = map;
}
/* Parse map entry. */
memset(&submsg, 0, size);
CHK(upb_decode_msgfield(d, submsg_ptr, entry, len));
/* Insert into map. */
if (map->key_size_lg2 == UPB_MAPTYPE_STRING) {
const upb_strview* key_view = (const upb_strview*)submsg_ptr;
key = key_view->data;
key_size = key_view->size;
} else {
key = submsg_ptr;
key_size = 1 << map->key_size_lg2;
}
if (map->val_size_lg2 == UPB_MAPTYPE_STRING) {
upb_strview* val_view = upb_arena_malloc(d->arena, sizeof(*val_view));
CHK(val_view);
memcpy(val_view, submsg_ptr + sizeof(upb_strview), sizeof(upb_strview));
memset(&val, 0, sizeof(val));
memcpy(&val, &val_view, sizeof(void*));
} else {
memcpy(&val, submsg_ptr + sizeof(upb_strview), 8);
}
if (!upb_strtable_lookup2(&map->table, key, key_size, NULL)) {
upb_strtable_insert3(&map->table, key, key_size, val, alloc);
}
return true;
}
static bool upb_decode_delimitedfield(upb_decstate *d, upb_decframe *frame,
const upb_msglayout_field *field) {
int len;
@ -472,12 +529,7 @@ static bool upb_decode_delimitedfield(upb_decstate *d, upb_decframe *frame,
if (field->label == UPB_LABEL_REPEATED) {
return upb_decode_toarray(d, frame, field, len);
} else if (field->label == UPB_LABEL_MAP) {
/* Max map entry size is string key/val. */
char submsg[sizeof(upb_strview) * 2];
const upb_msglayout *layout = frame->layout->submsgs[field->submsg_index];
CHK(upb_decode_msgfield(d, &submsg, layout, len));
/* TODO: insert into map. */
return true;
return upb_decode_mapfield(d, frame, field, len);
} else {
switch (field->descriptortype) {
case UPB_DESCRIPTOR_TYPE_STRING:

7
upb/def.c
Unescape View File

@ -868,7 +868,12 @@ static size_t upb_msgval_sizeof(upb_fieldtype_t type) {
}
static uint8_t upb_msg_fielddefsize(const upb_fielddef *f) {
if (upb_fielddef_isseq(f)) {
if (upb_msgdef_mapentry(upb_fielddef_containingtype(f))) {
// Map entries aren't actually stored, they are only used during parsing.
// For parsing, it helps a lot if all map entry messages have the same
// layout.
return sizeof(upb_strview);
} else if (upb_fielddef_isseq(f)) {
return sizeof(void*);
} else {
return upb_msgval_sizeof(upb_fielddef_type(f));

9
upb/encode.c
Unescape View File

@ -330,10 +330,13 @@ static bool upb_encode_map(upb_encstate *e, const char *field_mem,
size_t size;
upb_strview key = upb_strtable_iter_key(&i);
const upb_value val = upb_strtable_iter_value(&i);
const void* keyp =
map->key_size_lg2 == UPB_MAPTYPE_STRING ? (void*)&key : key.data;
const void* valp =
map->val_size_lg2 == UPB_MAPTYPE_STRING ? upb_value_getptr(val) : &val;
/* XXX; string key/value */
CHK(upb_encode_scalarfield(e, &val, entry, val_field, false));
CHK(upb_encode_scalarfield(e, key.data, entry, key_field, false));
CHK(upb_encode_scalarfield(e, valp, entry, val_field, false));
CHK(upb_encode_scalarfield(e, keyp, entry, key_field, false));
size = (e->limit - e->ptr) - pre_len;
CHK(upb_put_varint(e, size));
CHK(upb_put_tag(e, f->number, UPB_WIRE_TYPE_DELIMITED));

17
upb/msg.c
Unescape View File

@ -9,23 +9,6 @@
/** upb_msg *******************************************************************/
/* Internal members of a upb_msg. We can change this without breaking binary
* compatibility. We put these before the user's data. The user's upb_msg*
* points after the upb_msg_internal. */
/* Used when a message is not extendable. */
typedef struct {
char *unknown;
size_t unknown_len;
size_t unknown_size;
} upb_msg_internal;
/* Used when a message is extendable. */
typedef struct {
upb_inttable *extdict;
upb_msg_internal base;
} upb_msg_internal_withext;
static char _upb_fieldtype_to_sizelg2[12] = {
0,
0, /* UPB_TYPE_BOOL */

17
upb/msg.h
Unescape View File

@ -55,7 +55,22 @@ typedef struct upb_msglayout {
/** upb_msg *******************************************************************/
/* Representation is in msg.c for now. */
/* Internal members of a upb_msg. We can change this without breaking binary
* compatibility. We put these before the user's data. The user's upb_msg*
* points after the upb_msg_internal. */
/* Used when a message is not extendable. */
typedef struct {
char *unknown;
size_t unknown_len;
size_t unknown_size;
} upb_msg_internal;
/* Used when a message is extendable. */
typedef struct {
upb_inttable *extdict;
upb_msg_internal base;
} upb_msg_internal_withext;
/* Maps upb_fieldtype_t -> memory size. */
extern char _upb_fieldtype_to_size[12];

1
upb/reflection.c
Unescape View File

@ -223,7 +223,6 @@ bool upb_map_get(const upb_map *map, upb_msgval key, upb_msgval *val) {
ret = upb_strtable_lookup2(&map->table, strkey.data, strkey.size, &tabval);
if (ret) {
*val = upb_map_fromvalue(map->val_size_lg2, tabval);
memcpy(val, &tabval, sizeof(tabval));
}
return ret;

6
upb/reflection.h
Unescape View File

@ -1,6 +1,6 @@
#ifndef UPB_LEGACY_MSG_REFLECTION_H_
#define UPB_LEGACY_MSG_REFLECTION_H_
#ifndef UPB_REFLECTION_H_
#define UPB_REFLECTION_H_
#include "upb/def.h"
#include "upb/msg.h"
@ -134,4 +134,4 @@ bool upb_mapiter_isequal(const upb_mapiter *i1, const upb_mapiter *i2);
#include "upb/port_undef.inc"
#endif /* UPB_LEGACY_MSG_REFLECTION_H_ */
#endif /* UPB_REFLECTION_H_ */

2
upb/upb.h
Unescape View File

@ -358,8 +358,6 @@ typedef enum {
UPB_DESCRIPTOR_TYPE_SINT64 = 18
} upb_descriptortype_t;
extern const uint8_t upb_desctype_to_fieldtype[];
#include "upb/port_undef.inc"
#endif /* UPB_H_ */

11
upbc/message_layout.cc
Unescape View File

@ -1,5 +1,6 @@
#include "upbc/message_layout.h"
#include "google/protobuf/descriptor.pb.h"
namespace upbc {
@ -25,12 +26,18 @@ MessageLayout::Size MessageLayout::Place(
bool MessageLayout::HasHasbit(const protobuf::FieldDescriptor* field) {
return field->file()->syntax() == protobuf::FileDescriptor::SYNTAX_PROTO2 &&
field->label() != protobuf::FieldDescriptor::LABEL_REPEATED &&
!field->containing_oneof();
!field->containing_oneof() &&
!field->containing_type()->options().map_entry();
}
MessageLayout::SizeAndAlign MessageLayout::SizeOf(
const protobuf::FieldDescriptor* field) {
if (field->is_repeated()) {
if (field->containing_type()->options().map_entry()) {
// Map entries aren't actually stored, they are only used during parsing.
// For parsing, it helps a lot if all map entry messages have the same
// layout.
return {{8, 16}, {4, 8}}; // upb_stringview
} else if (field->is_repeated()) {
return {{4, 8}, {4, 8}}; // Pointer to array object.
} else {
return SizeOfUnwrapped(field);

Write Preview
Loading…
Cancel
Save