From b39dbb9d4010f6b6e11a5100a73e03b601cf72b9 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 12 Dec 2018 06:34:11 +0000 Subject: [PATCH 1/5] Add support for encoding Any in json --- tools/dump_cinit.lua | 1 + upb/def.h | 4 ++++ upb/json/parser.c | 32 +++++++++++++------------ upb/json/parser.rl | 8 +++---- upb/json/printer.c | 56 +++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/tools/dump_cinit.lua b/tools/dump_cinit.lua index 93ee12e57f..34d9dd1ec6 100644 --- a/tools/dump_cinit.lua +++ b/tools/dump_cinit.lua @@ -350,6 +350,7 @@ end local function well_known_type(m) local type_map = {} + type_map["google.protobuf.Any"] = "UPB_WELLKNOWN_ANY" type_map["google.protobuf.Duration"] = "UPB_WELLKNOWN_DURATION" type_map["google.protobuf.Timestamp"] = "UPB_WELLKNOWN_TIMESTAMP" type_map["google.protobuf.Value"] = "UPB_WELLKNOWN_VALUE" diff --git a/upb/def.h b/upb/def.h index c9ed1cf6e9..98b458ab0c 100644 --- a/upb/def.h +++ b/upb/def.h @@ -689,6 +689,10 @@ typedef upb_strtable_iter upb_msg_oneof_iter; #define UPB_MAPENTRY_KEY 1 #define UPB_MAPENTRY_VALUE 2 +/* Well-known field tag numbers for Any messages. */ +#define UPB_ANY_TYPE 1 +#define UPB_ANY_VALUE 2 + /* Well-known field tag numbers for timestamp messages. */ #define UPB_DURATION_SECONDS 1 #define UPB_DURATION_NANOS 2 diff --git a/upb/json/parser.c b/upb/json/parser.c index 83590a1fa2..28d7d8878b 100644 --- a/upb/json/parser.c +++ b/upb/json/parser.c @@ -1301,15 +1301,15 @@ static bool end_stringval_nontop(upb_json_parser *p) { return true; } - if (p->top->is_any) { - return end_any_stringval(p); - } - if (p->top->f == NULL) { multipart_end(p); return true; } + if (p->top->is_any) { + return end_any_stringval(p); + } + switch (upb_fielddef_type(p->top->f)) { case UPB_TYPE_BYTES: if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), @@ -2768,7 +2768,7 @@ _match: break; case 2: #line 2429 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 23;goto _again;} } + { p--; {stack[top++] = cs; cs = 23; goto _again;} } break; case 3: #line 2433 "upb/json/parser.rl" @@ -2842,17 +2842,17 @@ _match: #line 2488 "upb/json/parser.rl" { if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { - {stack[top++] = cs; cs = 47;goto _again;} + {stack[top++] = cs; cs = 47; goto _again;} } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { - {stack[top++] = cs; cs = 40;goto _again;} + {stack[top++] = cs; cs = 40; goto _again;} } else { - {stack[top++] = cs; cs = 32;goto _again;} + {stack[top++] = cs; cs = 32; goto _again;} } } break; case 21: #line 2499 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 75;goto _again;} } + { p--; {stack[top++] = cs; cs = 75; goto _again;} } break; case 22: #line 2504 "upb/json/parser.rl" @@ -2964,9 +2964,7 @@ _again: switch ( *__acts++ ) { case 0: #line 2425 "upb/json/parser.rl" - { p--; {cs = stack[--top]; if ( p == pe ) - goto _test_eof; -goto _again;} } + { p--; {cs = stack[--top]; goto _again;} } break; case 30: #line 2556 "upb/json/parser.rl" @@ -2988,7 +2986,7 @@ goto _again;} } #line 2568 "upb/json/parser.rl" { end_subobject_full(parser); } break; -#line 2992 "upb/json/parser.c" +#line 2990 "upb/json/parser.c" } } } @@ -3027,7 +3025,11 @@ static bool end(void *closure, const void *hd) { parse(parser, hd, &eof_ch, 0, NULL); - return parser->current_state >= 103; + return parser->current_state >= +#line 3030 "upb/json/parser.c" +103 +#line 2631 "upb/json/parser.rl" +; } static void json_parser_reset(upb_json_parser *p) { @@ -3044,7 +3046,7 @@ static void json_parser_reset(upb_json_parser *p) { /* Emit Ragel initialization of the parser. */ -#line 3048 "upb/json/parser.c" +#line 3050 "upb/json/parser.c" { cs = json_start; top = 0; diff --git a/upb/json/parser.rl b/upb/json/parser.rl index a7bdb3fe5d..e5845153a8 100644 --- a/upb/json/parser.rl +++ b/upb/json/parser.rl @@ -1299,15 +1299,15 @@ static bool end_stringval_nontop(upb_json_parser *p) { return true; } - if (p->top->is_any) { - return end_any_stringval(p); - } - if (p->top->f == NULL) { multipart_end(p); return true; } + if (p->top->is_any) { + return end_any_stringval(p); + } + switch (upb_fielddef_type(p->top->f)) { case UPB_TYPE_BYTES: if (!base64_push(p, getsel_for_handlertype(p, UPB_HANDLER_STRING), diff --git a/upb/json/printer.c b/upb/json/printer.c index fe306d45ac..7e6393038a 100644 --- a/upb/json/printer.c +++ b/upb/json/printer.c @@ -692,6 +692,14 @@ void printer_sethandlers_mapentry(const void *closure, bool preserve_fieldnames, upb_handlerattr_uninit(&empty_attr); } +static void *scalar_startstr_any_typeurl(void *closure, const void *handler_data, + size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(handler_data); + UPB_UNUSED(size_hint); + return p; +} + static bool putseconds(void *closure, const void *handler_data, int64_t seconds) { upb_json_printer *p = closure; @@ -920,6 +928,51 @@ static bool printer_endmsg_noframe( return true; } +static void *scalar_startstr_onlykey( + void *closure, const void *handler_data, size_t size_hint) { + upb_json_printer *p = closure; + UPB_UNUSED(size_hint); + CHK(putkey(closure, handler_data)); + return p; +} + +/* Set up handlers for an Any submessage. */ +void printer_sethandlers_any(const void *closure, upb_handlers *h) { + const upb_msgdef *md = upb_handlers_msgdef(h); + + const upb_fielddef* type_field = upb_msgdef_itof(md, UPB_ANY_TYPE); + const upb_fielddef* value_field = upb_msgdef_itof(md, UPB_ANY_VALUE); + + upb_handlerattr empty_attr = UPB_HANDLERATTR_INITIALIZER; + + /* type_url's json name is "@type" */ + upb_handlerattr type_name_attr = UPB_HANDLERATTR_INITIALIZER; + upb_handlerattr value_name_attr = UPB_HANDLERATTR_INITIALIZER; + strpc *type_url_json_name = upb_gmalloc(sizeof(*type_url_json_name)); + strpc *value_json_name = upb_gmalloc(sizeof(*type_url_json_name)); + + type_url_json_name->ptr = upb_gstrdup("@type"); + type_url_json_name->len = strlen(type_url_json_name->ptr); + upb_handlers_addcleanup(h, type_url_json_name, freestrpc); + upb_handlerattr_sethandlerdata(&type_name_attr, type_url_json_name); + + value_json_name->ptr = upb_gstrdup("value"); + value_json_name->len = strlen(value_json_name->ptr); + upb_handlers_addcleanup(h, value_json_name, freestrpc); + upb_handlerattr_sethandlerdata(&value_name_attr, value_json_name); + + /* Set up handlers. */ + upb_handlers_setstartmsg(h, printer_startmsg, &empty_attr); + upb_handlers_setendmsg(h, printer_endmsg, &empty_attr); + + upb_handlers_setstartstr(h, type_field, scalar_startstr, &type_name_attr); + upb_handlers_setstring(h, type_field, scalar_str, &empty_attr); + upb_handlers_setendstr(h, type_field, scalar_endstr, &empty_attr); + + upb_handlers_setstartstr(h, value_field, scalar_startstr_onlykey, + &value_name_attr); +} + /* Set up handlers for a duration submessage. */ void printer_sethandlers_duration(const void *closure, upb_handlers *h) { const upb_msgdef *md = upb_handlers_msgdef(h); @@ -1074,7 +1127,8 @@ void printer_sethandlers(const void *closure, upb_handlers *h) { case UPB_WELLKNOWN_UNSPECIFIED: break; case UPB_WELLKNOWN_ANY: - break; + printer_sethandlers_any(closure, h); + return; case UPB_WELLKNOWN_DURATION: printer_sethandlers_duration(closure, h); return; From 915806ea8d0789df6155f6f510e34bdc0a17d093 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 12 Dec 2018 06:40:32 +0000 Subject: [PATCH 2/5] Remove unused code --- upb/json/printer.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/upb/json/printer.c b/upb/json/printer.c index 7e6393038a..6611581c7d 100644 --- a/upb/json/printer.c +++ b/upb/json/printer.c @@ -692,14 +692,6 @@ void printer_sethandlers_mapentry(const void *closure, bool preserve_fieldnames, upb_handlerattr_uninit(&empty_attr); } -static void *scalar_startstr_any_typeurl(void *closure, const void *handler_data, - size_t size_hint) { - upb_json_printer *p = closure; - UPB_UNUSED(handler_data); - UPB_UNUSED(size_hint); - return p; -} - static bool putseconds(void *closure, const void *handler_data, int64_t seconds) { upb_json_printer *p = closure; From 375a25423ee0e5bf5bd53ae965a534e289335582 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 12 Dec 2018 06:43:00 +0000 Subject: [PATCH 3/5] Mark unused parameter --- upb/json/printer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upb/json/printer.c b/upb/json/printer.c index 6611581c7d..9805c23a39 100644 --- a/upb/json/printer.c +++ b/upb/json/printer.c @@ -963,6 +963,8 @@ void printer_sethandlers_any(const void *closure, upb_handlers *h) { upb_handlers_setstartstr(h, value_field, scalar_startstr_onlykey, &value_name_attr); + + UPB_UNUSED(closure); } /* Set up handlers for a duration submessage. */ From 99600b4a0e483c41144d876742abc09a95ff985b Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Thu, 13 Dec 2018 13:44:25 -0800 Subject: [PATCH 4/5] Fix generated file test --- upb/json/parser.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/upb/json/parser.c b/upb/json/parser.c index 28d7d8878b..08056d39af 100644 --- a/upb/json/parser.c +++ b/upb/json/parser.c @@ -2768,7 +2768,7 @@ _match: break; case 2: #line 2429 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 23; goto _again;} } + { p--; {stack[top++] = cs; cs = 23;goto _again;} } break; case 3: #line 2433 "upb/json/parser.rl" @@ -2842,17 +2842,17 @@ _match: #line 2488 "upb/json/parser.rl" { if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { - {stack[top++] = cs; cs = 47; goto _again;} + {stack[top++] = cs; cs = 47;goto _again;} } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { - {stack[top++] = cs; cs = 40; goto _again;} + {stack[top++] = cs; cs = 40;goto _again;} } else { - {stack[top++] = cs; cs = 32; goto _again;} + {stack[top++] = cs; cs = 32;goto _again;} } } break; case 21: #line 2499 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 75; goto _again;} } + { p--; {stack[top++] = cs; cs = 75;goto _again;} } break; case 22: #line 2504 "upb/json/parser.rl" @@ -2964,7 +2964,9 @@ _again: switch ( *__acts++ ) { case 0: #line 2425 "upb/json/parser.rl" - { p--; {cs = stack[--top]; goto _again;} } + { p--; {cs = stack[--top]; if ( p == pe ) + goto _test_eof; +goto _again;} } break; case 30: #line 2556 "upb/json/parser.rl" @@ -2986,7 +2988,7 @@ _again: #line 2568 "upb/json/parser.rl" { end_subobject_full(parser); } break; -#line 2990 "upb/json/parser.c" +#line 2992 "upb/json/parser.c" } } } @@ -3025,11 +3027,7 @@ static bool end(void *closure, const void *hd) { parse(parser, hd, &eof_ch, 0, NULL); - return parser->current_state >= -#line 3030 "upb/json/parser.c" -103 -#line 2631 "upb/json/parser.rl" -; + return parser->current_state >= 103; } static void json_parser_reset(upb_json_parser *p) { @@ -3046,7 +3044,7 @@ static void json_parser_reset(upb_json_parser *p) { /* Emit Ragel initialization of the parser. */ -#line 3050 "upb/json/parser.c" +#line 3048 "upb/json/parser.c" { cs = json_start; top = 0; From 18563bee4c9822a1ee42bb1d93024981777eea41 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 19 Dec 2018 19:36:30 +0000 Subject: [PATCH 5/5] Fix comments --- upb/json/printer.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/upb/json/printer.c b/upb/json/printer.c index 9805c23a39..5ed79a0f3d 100644 --- a/upb/json/printer.c +++ b/upb/json/printer.c @@ -69,6 +69,15 @@ strpc *newstrpc(upb_handlers *h, const upb_fielddef *f, return ret; } +/* Convert a null-terminated const char* to a string piece. */ +strpc *newstrpc_str(upb_handlers *h, const char * str) { + strpc * ret = upb_gmalloc(sizeof(*ret)); + ret->ptr = upb_gstrdup(str); + ret->len = strlen(str); + upb_handlers_addcleanup(h, ret, freestrpc); + return ret; +} + /* ------------ JSON string printing: values, maps, arrays ------------------ */ static void print_data( @@ -940,17 +949,10 @@ void printer_sethandlers_any(const void *closure, upb_handlers *h) { /* type_url's json name is "@type" */ upb_handlerattr type_name_attr = UPB_HANDLERATTR_INITIALIZER; upb_handlerattr value_name_attr = UPB_HANDLERATTR_INITIALIZER; - strpc *type_url_json_name = upb_gmalloc(sizeof(*type_url_json_name)); - strpc *value_json_name = upb_gmalloc(sizeof(*type_url_json_name)); + strpc *type_url_json_name = newstrpc_str(h, "@type"); + strpc *value_json_name = newstrpc_str(h, "value"); - type_url_json_name->ptr = upb_gstrdup("@type"); - type_url_json_name->len = strlen(type_url_json_name->ptr); - upb_handlers_addcleanup(h, type_url_json_name, freestrpc); upb_handlerattr_sethandlerdata(&type_name_attr, type_url_json_name); - - value_json_name->ptr = upb_gstrdup("value"); - value_json_name->len = strlen(value_json_name->ptr); - upb_handlers_addcleanup(h, value_json_name, freestrpc); upb_handlerattr_sethandlerdata(&value_name_attr, value_json_name); /* Set up handlers. */ @@ -961,6 +963,9 @@ void printer_sethandlers_any(const void *closure, upb_handlers *h) { upb_handlers_setstring(h, type_field, scalar_str, &empty_attr); upb_handlers_setendstr(h, type_field, scalar_endstr, &empty_attr); + /* This is not the full and correct JSON encoding for the Any value field. It + * requires further processing by the wrapper code based on the type URL. + */ upb_handlers_setstartstr(h, value_field, scalar_startstr_onlykey, &value_name_attr);