From 6b357607bd35fef98ad3c0f3312845b9798efd0f Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 8 Dec 2020 16:58:30 -0800 Subject: [PATCH 1/3] Fixed bugs in JSON encoding with UPB_JSONENC_EMITDEFAULTS. Previously the code would crash on certain empty fields. --- upb/json_encode.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/upb/json_encode.c b/upb/json_encode.c index 628d1c194a..9217c65b25 100644 --- a/upb/json_encode.c +++ b/upb/json_encode.c @@ -594,7 +594,7 @@ static void jsonenc_mapkey(jsonenc *e, upb_msgval val, const upb_fielddef *f) { static void jsonenc_array(jsonenc *e, const upb_array *arr, const upb_fielddef *f) { size_t i; - size_t size = upb_array_size(arr); + size_t size = arr ? upb_array_size(arr) : 0; bool first = true; jsonenc_putstr(e, "["); @@ -616,10 +616,12 @@ static void jsonenc_map(jsonenc *e, const upb_map *map, const upb_fielddef *f) { jsonenc_putstr(e, "{"); - while (upb_mapiter_next(map, &iter)) { - jsonenc_putsep(e, ",", &first); - jsonenc_mapkey(e, upb_mapiter_key(map, iter), key_f); - jsonenc_scalar(e, upb_mapiter_value(map, iter), val_f); + if (map) { + while (upb_mapiter_next(map, &iter)) { + jsonenc_putsep(e, ",", &first); + jsonenc_mapkey(e, upb_mapiter_key(map, iter), key_f); + jsonenc_scalar(e, upb_mapiter_value(map, iter), val_f); + } } jsonenc_putstr(e, "}"); @@ -659,7 +661,9 @@ static void jsonenc_msgfields(jsonenc *e, const upb_msg *msg, int n = upb_msgdef_fieldcount(m); for (i = 0; i < n; i++) { f = upb_msgdef_field(m, i); - jsonenc_fieldval(e, f, upb_msg_get(msg, f), &first); + if (!upb_fielddef_haspresence(f) || upb_msg_has(msg, f)) { + jsonenc_fieldval(e, f, upb_msg_get(msg, f), &first); + } } } else { /* Iterate over non-empty fields. */ From 695b7f46171ba7dbe6e5817c4047b23c68bd5e6b Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 8 Dec 2020 17:00:39 -0800 Subject: [PATCH 2/3] Added code to test UPB_JSONENC_EMITDEFAULTS. --- BUILD | 5 +- tests/bindings/lua/test_upb.lua | 6 + upb/bindings/lua/BUILD | 1 + upb/bindings/lua/def.c | 2 +- upb/bindings/lua/msg.c | 200 ++++++++++++++++++++------------ upb/bindings/lua/upb.h | 2 +- 6 files changed, 136 insertions(+), 80 deletions(-) diff --git a/BUILD b/BUILD index 5ba41c9edd..5a9f3b0e72 100644 --- a/BUILD +++ b/BUILD @@ -187,7 +187,10 @@ cc_library( "upb/json_encode.h", ], copts = UPB_DEFAULT_COPTS, - visibility = ["//tests:__pkg__"], + visibility = [ + "//tests:__pkg__", + "//upb/bindings/lua:__pkg__", + ], deps = [ ":port", ":reflection", diff --git a/tests/bindings/lua/test_upb.lua b/tests/bindings/lua/test_upb.lua index b966fcc4be..22c774851d 100644 --- a/tests/bindings/lua/test_upb.lua +++ b/tests/bindings/lua/test_upb.lua @@ -695,6 +695,12 @@ function test_encode_skipunknown() assert_true(#upb.encode(empty_with_unknown, {upb.ENCODE_SKIPUNKNOWN}) == 0) end +function test_json_emit_defaults() + local msg = test_messages_proto3.TestAllTypesProto3() + local json = upb.json_encode(msg, {upb.JSONENC_EMITDEFAULTS}) + print(json) +end + function test_gc() local top = test_messages_proto3.TestAllTypesProto3() local n = 100 diff --git a/upb/bindings/lua/BUILD b/upb/bindings/lua/BUILD index cf1aa76371..3b794a85a3 100644 --- a/upb/bindings/lua/BUILD +++ b/upb/bindings/lua/BUILD @@ -19,6 +19,7 @@ cc_library( copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ + "//:json", "//:reflection", "//:textformat", "//:upb", diff --git a/upb/bindings/lua/def.c b/upb/bindings/lua/def.c index 5a4d42c4ac..a1e8720583 100644 --- a/upb/bindings/lua/def.c +++ b/upb/bindings/lua/def.c @@ -497,7 +497,7 @@ static int lupb_msgdef_tostring(lua_State *L) { } static const struct luaL_Reg lupb_msgdef_mm[] = { - {"__call", lupb_msg_pushnew}, + {"__call", lupb_msgdef_call}, {"__index", lupb_msgdef_index}, {"__len", lupb_msgdef_fieldcount}, {"__tostring", lupb_msgdef_tostring}, diff --git a/upb/bindings/lua/msg.c b/upb/bindings/lua/msg.c index 792d5f8229..c0c5bb9933 100644 --- a/upb/bindings/lua/msg.c +++ b/upb/bindings/lua/msg.c @@ -12,11 +12,12 @@ #include "lauxlib.h" #include "upb/bindings/lua/upb.h" +#include "upb/json_decode.h" +#include "upb/json_encode.h" +#include "upb/port_def.inc" #include "upb/reflection.h" #include "upb/text_encode.h" -#include "upb/port_def.inc" - /* * Message/Map/Array objects. These objects form a directed graph: a message * can contain submessages, arrays, and maps, which can then point to other @@ -637,6 +638,20 @@ static const upb_fielddef *lupb_msg_checkfield(lua_State *L, int msg, return f; } +upb_msg *lupb_msg_pushnew(lua_State *L, int narg) { + const upb_msgdef *m = lupb_msgdef_check(L, narg); + lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), 2, LUPB_MSG); + upb_arena *arena = lupb_arena_pushnew(L); + + lua_setiuservalue(L, -2, LUPB_ARENA_INDEX); + lua_pushvalue(L, 1); + lua_setiuservalue(L, -2, LUPB_MSGDEF_INDEX); + + lmsg->msg = upb_msg_new(m, arena); + lupb_cacheset(L, lmsg->msg); + return lmsg->msg; +} + /** * lupb_msg_newmsgwrapper() * @@ -726,28 +741,19 @@ static void lupb_msg_typechecksubmsg(lua_State *L, int narg, int msgarg, /* lupb_msg Public API */ /** - * lupb_msg_pushnew + * lupb_msgdef_call * * Handles: * new_msg = MessageClass() * new_msg = MessageClass{foo = "bar", baz = 3, quux = {foo = 3}} */ -int lupb_msg_pushnew(lua_State *L) { - int argcount = lua_gettop(L); - const upb_msgdef *m = lupb_msgdef_check(L, 1); - lupb_msg *lmsg = lupb_newuserdata(L, sizeof(lupb_msg), 2, LUPB_MSG); - upb_arena *arena = lupb_arena_pushnew(L); - - lua_setiuservalue(L, -2, LUPB_ARENA_INDEX); - lua_pushvalue(L, 1); - lua_setiuservalue(L, -2, LUPB_MSGDEF_INDEX); - - lmsg->msg = upb_msg_new(m, arena); - lupb_cacheset(L, lmsg->msg); +int lupb_msgdef_call(lua_State *L) { + int arg_count = lua_gettop(L); + lupb_msg_pushnew(L, 1); - if (argcount > 1) { + if (arg_count > 1) { /* Set initial fields from table. */ - int msg = lua_gettop(L); + int msg = arg_count + 1; lua_pushnil(L); while (lua_next(L, 2) != 0) { lua_pushvalue(L, -2); /* now stack is key, val, key */ @@ -883,6 +889,19 @@ static const struct luaL_Reg lupb_msg_mm[] = { /* lupb_msg toplevel **********************************************************/ +static int lupb_getoptions(lua_State *L, int narg) { + int options = 0; + if (lua_gettop(L) >= narg) { + size_t len = lua_rawlen(L, narg); + for (size_t i = 1; i <= len; i++) { + lua_rawgeti(L, narg, i); + options |= lupb_checkuint32(L, -1); + lua_pop(L, 1); + } + } + return options; +} + /** * lupb_decode() * @@ -894,26 +913,16 @@ static int lupb_decode(lua_State *L) { const upb_msgdef *m = lupb_msgdef_check(L, 1); const char *pb = lua_tolstring(L, 2, &len); const upb_msglayout *layout = upb_msgdef_layout(m); + upb_msg *msg = lupb_msg_pushnew(L, 1); + upb_arena *arena = lupb_arenaget(L, -1); char *buf; - upb_msg *msg; - upb_arena *arena; bool ok; - /* Create message. */ - lua_pushcfunction(L, &lupb_msg_pushnew); - lua_pushvalue(L, 1); - lua_call(L, 1, 1); - msg = lupb_msg_check(L, -1); - - lua_getiuservalue(L, -1, LUPB_ARENA_INDEX); - arena = lupb_arena_check(L, -1); - lua_pop(L, 1); - /* Copy input data to arena, message will reference it. */ buf = upb_arena_malloc(arena, len); memcpy(buf, pb, len); - ok = upb_decode(buf, len, msg, layout, arena); + ok = _upb_decode(buf, len, msg, layout, arena, UPB_DECODE_ALIAS); if (!ok) { lua_pushstring(L, "Error decoding protobuf."); @@ -924,38 +933,81 @@ static int lupb_decode(lua_State *L) { } /** - * lupb_msg_textencode() + * lupb_encode() * * Handles: - * text_string = upb.text_encode(msg, {upb.TXTENC_SINGLELINE}) + * bin_string = upb.encode(msg) */ -static int lupb_textencode(lua_State *L) { - int argcount = lua_gettop(L); - upb_msg *msg = lupb_msg_check(L, 1); - const upb_msgdef *m; - char buf[1024]; +static int lupb_encode(lua_State *L) { + const upb_msg *msg = lupb_msg_check(L, 1); + const upb_msgdef *m = lupb_msg_getmsgdef(L, 1); + const upb_msglayout *layout = upb_msgdef_layout(m); + int options = lupb_getoptions(L, 2); + upb_arena *arena; size_t size; - int options = 0; + char *result; - lua_getiuservalue(L, 1, LUPB_MSGDEF_INDEX); - m = lupb_msgdef_check(L, -1); + arena = lupb_arena_pushnew(L); + result = upb_encode_ex(msg, (const void*)layout, options, arena, &size); - if (argcount > 1) { - size_t len = lua_rawlen(L, 2); - for (size_t i = 1; i <= len; i++) { - lua_rawgeti(L, 2, i); - options |= lupb_checkuint32(L, -1); - lua_pop(L, 1); - } + if (!result) { + lua_pushstring(L, "Error encoding protobuf."); + return lua_error(L); } - size = upb_text_encode(msg, m, NULL, options, buf, sizeof(buf)); + lua_pushlstring(L, result, size); + + return 1; +} + +/** + * lupb_jsondecode() + * + * Handles: + * text_string = upb.json_decode(MessageClass, json_str, {upb.JSONDEC_IGNOREUNKNOWN}) + */ +static int lupb_jsondecode(lua_State *L) { + size_t len; + const upb_msgdef *m = lupb_msgdef_check(L, 1); + const char *json = lua_tolstring(L, 2, &len); + int options = lupb_getoptions(L, 3); + upb_msg *msg; + upb_arena *arena; + upb_status status; + + msg = lupb_msg_pushnew(L, 1); + arena = lupb_arenaget(L, -1); + upb_status_clear(&status); + upb_json_decode(json, len, msg, m, NULL, options, arena, &status); + lupb_checkstatus(L, &status); + + return 1; +} + +/** + * lupb_jsonencode() + * + * Handles: + * text_string = upb.json_encode(msg, {upb.JSONENC_EMITDEFAULTS}) + */ +static int lupb_jsonencode(lua_State *L) { + upb_msg *msg = lupb_msg_check(L, 1); + const upb_msgdef *m = lupb_msg_getmsgdef(L, 1); + int options = lupb_getoptions(L, 2); + char buf[1024]; + size_t size; + upb_status status; + + upb_status_clear(&status); + size = upb_json_encode(msg, m, NULL, options, buf, sizeof(buf), &status); + lupb_checkstatus(L, &status); if (size < sizeof(buf)) { lua_pushlstring(L, buf, size); } else { char *ptr = malloc(size + 1); - upb_text_encode(msg, m, NULL, options, ptr, size + 1); + upb_json_encode(msg, m, NULL, options, ptr, size + 1, &status); + lupb_checkstatus(L, &status); lua_pushlstring(L, ptr, size); free(ptr); } @@ -964,42 +1016,29 @@ static int lupb_textencode(lua_State *L) { } /** - * lupb_encode() + * lupb_textencode() * * Handles: - * bin_string = upb.encode(msg) + * text_string = upb.text_encode(msg, {upb.TXTENC_SINGLELINE}) */ -static int lupb_encode(lua_State *L) { - int argcount = lua_gettop(L); - const upb_msg *msg = lupb_msg_check(L, 1); - const upb_msglayout *layout; - upb_arena *arena = lupb_arena_pushnew(L); +static int lupb_textencode(lua_State *L) { + upb_msg *msg = lupb_msg_check(L, 1); + const upb_msgdef *m = lupb_msg_getmsgdef(L, 1); + int options = lupb_getoptions(L, 2); + char buf[1024]; size_t size; - char *result; - int options = 0; - - if (argcount > 1) { - size_t len = lua_rawlen(L, 2); - for (size_t i = 1; i <= len; i++) { - lua_rawgeti(L, 2, i); - options |= lupb_checkuint32(L, -1); - lua_pop(L, 1); - } - } - lua_getiuservalue(L, 1, LUPB_MSGDEF_INDEX); - layout = upb_msgdef_layout(lupb_msgdef_check(L, -1)); - lua_pop(L, 1); - - result = upb_encode_ex(msg, (const void*)layout, options, arena, &size); + size = upb_text_encode(msg, m, NULL, options, buf, sizeof(buf)); - if (!result) { - lua_pushstring(L, "Error encoding protobuf."); - return lua_error(L); + if (size < sizeof(buf)) { + lua_pushlstring(L, buf, size); + } else { + char *ptr = malloc(size + 1); + upb_text_encode(msg, m, NULL, options, ptr, size + 1); + lua_pushlstring(L, ptr, size); + free(ptr); } - lua_pushlstring(L, result, size); - return 1; } @@ -1013,6 +1052,8 @@ static const struct luaL_Reg lupb_msg_toplevel_m[] = { {"Map", lupb_map_new}, {"decode", lupb_decode}, {"encode", lupb_encode}, + {"json_decode", lupb_jsondecode}, + {"json_encode", lupb_jsonencode}, {"text_encode", lupb_textencode}, {NULL, NULL} }; @@ -1032,5 +1073,10 @@ void lupb_msg_registertypes(lua_State *L) { lupb_setfieldi(L, "ENCODE_DETERMINISTIC", UPB_ENCODE_DETERMINISTIC); lupb_setfieldi(L, "ENCODE_SKIPUNKNOWN", UPB_ENCODE_SKIPUNKNOWN); + lupb_setfieldi(L, "JSONENC_EMITDEFAULTS", UPB_JSONENC_EMITDEFAULTS); + lupb_setfieldi(L, "JSONENC_PROTONAMES", UPB_JSONENC_PROTONAMES); + + lupb_setfieldi(L, "JSONDEC_IGNOREUNKNOWN", UPB_JSONDEC_IGNOREUNKNOWN); + lupb_cacheinit(L); } diff --git a/upb/bindings/lua/upb.h b/upb/bindings/lua/upb.h index 02ed341098..455752fd83 100644 --- a/upb/bindings/lua/upb.h +++ b/upb/bindings/lua/upb.h @@ -75,7 +75,7 @@ void lupb_def_registertypes(lua_State *L); /** From msg.c. ***************************************************************/ -int lupb_msg_pushnew(lua_State *L); +int lupb_msgdef_call(lua_State *L); upb_arena *lupb_arena_pushnew(lua_State *L); void lupb_msg_registertypes(lua_State *L); From 7a17493269a66d4dece71ac38bc7ed2931eb912b Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Tue, 8 Dec 2020 17:07:35 -0800 Subject: [PATCH 3/3] Removed print debugging. --- tests/bindings/lua/test_upb.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/bindings/lua/test_upb.lua b/tests/bindings/lua/test_upb.lua index 22c774851d..c82340388d 100644 --- a/tests/bindings/lua/test_upb.lua +++ b/tests/bindings/lua/test_upb.lua @@ -698,7 +698,6 @@ end function test_json_emit_defaults() local msg = test_messages_proto3.TestAllTypesProto3() local json = upb.json_encode(msg, {upb.JSONENC_EMITDEFAULTS}) - print(json) end function test_gc()