Ignore Unknown Enum Value (#280)

* Add tests for proto2

* Fix ignore unknown enum value

* Fix CheckConsistency

* Add note to mention clear field for unknown enum value

* Update Comment

* Fix generated test

* Fix generated test
pull/13171/head
Paul Yang 5 years ago committed by GitHub
parent 6c4acba610
commit c26e8db53d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 145
      generated_for_cmake/upb/json/parser.c
  2. 91
      tests/json/test_json.cc
  3. 7
      upb/json/parser.rl

@ -1372,7 +1372,12 @@ static bool end_stringval_nontop(upb_json_parser *p) {
upb_selector_t sel = parser_getsel(p);
upb_sink_putint32(p->top->sink, sel, int_val);
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
if (p->ignore_json_unknown) {
ok = true;
/* TODO(teboring): Should also clean this field. */
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
}
}
break;
@ -2577,11 +2582,11 @@ static bool does_fieldmask_end(upb_json_parser *p) {
* final state once, when the closing '"' is seen. */
#line 2782 "upb/json/parser.rl"
#line 2787 "upb/json/parser.rl"
#line 2585 "upb/json/parser.c"
#line 2590 "upb/json/parser.c"
static const char _json_actions[] = {
0, 1, 0, 1, 1, 1, 3, 1,
4, 1, 6, 1, 7, 1, 8, 1,
@ -2836,7 +2841,7 @@ static const int json_en_value_machine = 78;
static const int json_en_main = 1;
#line 2785 "upb/json/parser.rl"
#line 2790 "upb/json/parser.rl"
size_t parse(void *closure, const void *hd, const char *buf, size_t size,
const upb_bufhandle *handle) {
@ -2859,7 +2864,7 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size,
capture_resume(parser, buf);
#line 2863 "upb/json/parser.c"
#line 2868 "upb/json/parser.c"
{
int _klen;
unsigned int _trans;
@ -2934,147 +2939,147 @@ _match:
switch ( *_acts++ )
{
case 1:
#line 2590 "upb/json/parser.rl"
#line 2595 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 2:
#line 2592 "upb/json/parser.rl"
#line 2597 "upb/json/parser.rl"
{ p--; {stack[top++] = cs; cs = 23;goto _again;} }
break;
case 3:
#line 2596 "upb/json/parser.rl"
#line 2601 "upb/json/parser.rl"
{ start_text(parser, p); }
break;
case 4:
#line 2597 "upb/json/parser.rl"
#line 2602 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_text(parser, p)); }
break;
case 5:
#line 2603 "upb/json/parser.rl"
#line 2608 "upb/json/parser.rl"
{ start_hex(parser); }
break;
case 6:
#line 2604 "upb/json/parser.rl"
#line 2609 "upb/json/parser.rl"
{ hexdigit(parser, p); }
break;
case 7:
#line 2605 "upb/json/parser.rl"
#line 2610 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_hex(parser)); }
break;
case 8:
#line 2611 "upb/json/parser.rl"
#line 2616 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(escape(parser, p)); }
break;
case 9:
#line 2617 "upb/json/parser.rl"
#line 2622 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 10:
#line 2622 "upb/json/parser.rl"
#line 2627 "upb/json/parser.rl"
{ start_year(parser, p); }
break;
case 11:
#line 2623 "upb/json/parser.rl"
#line 2628 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_year(parser, p)); }
break;
case 12:
#line 2627 "upb/json/parser.rl"
#line 2632 "upb/json/parser.rl"
{ start_month(parser, p); }
break;
case 13:
#line 2628 "upb/json/parser.rl"
#line 2633 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_month(parser, p)); }
break;
case 14:
#line 2632 "upb/json/parser.rl"
#line 2637 "upb/json/parser.rl"
{ start_day(parser, p); }
break;
case 15:
#line 2633 "upb/json/parser.rl"
#line 2638 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_day(parser, p)); }
break;
case 16:
#line 2637 "upb/json/parser.rl"
#line 2642 "upb/json/parser.rl"
{ start_hour(parser, p); }
break;
case 17:
#line 2638 "upb/json/parser.rl"
#line 2643 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_hour(parser, p)); }
break;
case 18:
#line 2642 "upb/json/parser.rl"
#line 2647 "upb/json/parser.rl"
{ start_minute(parser, p); }
break;
case 19:
#line 2643 "upb/json/parser.rl"
#line 2648 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_minute(parser, p)); }
break;
case 20:
#line 2647 "upb/json/parser.rl"
#line 2652 "upb/json/parser.rl"
{ start_second(parser, p); }
break;
case 21:
#line 2648 "upb/json/parser.rl"
#line 2653 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_second(parser, p)); }
break;
case 22:
#line 2653 "upb/json/parser.rl"
#line 2658 "upb/json/parser.rl"
{ start_duration_base(parser, p); }
break;
case 23:
#line 2654 "upb/json/parser.rl"
#line 2659 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_duration_base(parser, p)); }
break;
case 24:
#line 2656 "upb/json/parser.rl"
#line 2661 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 25:
#line 2661 "upb/json/parser.rl"
#line 2666 "upb/json/parser.rl"
{ start_timestamp_base(parser); }
break;
case 26:
#line 2663 "upb/json/parser.rl"
#line 2668 "upb/json/parser.rl"
{ start_timestamp_fraction(parser, p); }
break;
case 27:
#line 2664 "upb/json/parser.rl"
#line 2669 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); }
break;
case 28:
#line 2666 "upb/json/parser.rl"
#line 2671 "upb/json/parser.rl"
{ start_timestamp_zone(parser, p); }
break;
case 29:
#line 2667 "upb/json/parser.rl"
#line 2672 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); }
break;
case 30:
#line 2669 "upb/json/parser.rl"
#line 2674 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 31:
#line 2674 "upb/json/parser.rl"
#line 2679 "upb/json/parser.rl"
{ start_fieldmask_path_text(parser, p); }
break;
case 32:
#line 2675 "upb/json/parser.rl"
#line 2680 "upb/json/parser.rl"
{ end_fieldmask_path_text(parser, p); }
break;
case 33:
#line 2680 "upb/json/parser.rl"
#line 2685 "upb/json/parser.rl"
{ start_fieldmask_path(parser); }
break;
case 34:
#line 2681 "upb/json/parser.rl"
#line 2686 "upb/json/parser.rl"
{ end_fieldmask_path(parser); }
break;
case 35:
#line 2687 "upb/json/parser.rl"
#line 2692 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 36:
#line 2692 "upb/json/parser.rl"
#line 2697 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) {
{stack[top++] = cs; cs = 47;goto _again;}
@ -3088,11 +3093,11 @@ _match:
}
break;
case 37:
#line 2705 "upb/json/parser.rl"
#line 2710 "upb/json/parser.rl"
{ p--; {stack[top++] = cs; cs = 78;goto _again;} }
break;
case 38:
#line 2710 "upb/json/parser.rl"
#line 2715 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
start_any_member(parser, p);
@ -3102,11 +3107,11 @@ _match:
}
break;
case 39:
#line 2717 "upb/json/parser.rl"
#line 2722 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_membername(parser)); }
break;
case 40:
#line 2720 "upb/json/parser.rl"
#line 2725 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
end_any_member(parser, p);
@ -3116,7 +3121,7 @@ _match:
}
break;
case 41:
#line 2731 "upb/json/parser.rl"
#line 2736 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
start_any_object(parser, p);
@ -3126,7 +3131,7 @@ _match:
}
break;
case 42:
#line 2740 "upb/json/parser.rl"
#line 2745 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
CHECK_RETURN_TOP(end_any_object(parser, p));
@ -3136,54 +3141,54 @@ _match:
}
break;
case 43:
#line 2752 "upb/json/parser.rl"
#line 2757 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_array(parser)); }
break;
case 44:
#line 2756 "upb/json/parser.rl"
#line 2761 "upb/json/parser.rl"
{ end_array(parser); }
break;
case 45:
#line 2761 "upb/json/parser.rl"
#line 2766 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_number(parser, p)); }
break;
case 46:
#line 2762 "upb/json/parser.rl"
#line 2767 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_number(parser, p)); }
break;
case 47:
#line 2764 "upb/json/parser.rl"
#line 2769 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_stringval(parser)); }
break;
case 48:
#line 2765 "upb/json/parser.rl"
#line 2770 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_stringval(parser)); }
break;
case 49:
#line 2767 "upb/json/parser.rl"
#line 2772 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, true)); }
break;
case 50:
#line 2769 "upb/json/parser.rl"
#line 2774 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, false)); }
break;
case 51:
#line 2771 "upb/json/parser.rl"
#line 2776 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_null(parser)); }
break;
case 52:
#line 2773 "upb/json/parser.rl"
#line 2778 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_subobject_full(parser)); }
break;
case 53:
#line 2774 "upb/json/parser.rl"
#line 2779 "upb/json/parser.rl"
{ end_subobject_full(parser); }
break;
case 54:
#line 2779 "upb/json/parser.rl"
#line 2784 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
#line 3187 "upb/json/parser.c"
#line 3192 "upb/json/parser.c"
}
}
@ -3200,32 +3205,32 @@ _again:
while ( __nacts-- > 0 ) {
switch ( *__acts++ ) {
case 0:
#line 2588 "upb/json/parser.rl"
#line 2593 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; if ( p == pe )
goto _test_eof;
goto _again;} }
break;
case 46:
#line 2762 "upb/json/parser.rl"
#line 2767 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_number(parser, p)); }
break;
case 49:
#line 2767 "upb/json/parser.rl"
#line 2772 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, true)); }
break;
case 50:
#line 2769 "upb/json/parser.rl"
#line 2774 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, false)); }
break;
case 51:
#line 2771 "upb/json/parser.rl"
#line 2776 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_null(parser)); }
break;
case 53:
#line 2774 "upb/json/parser.rl"
#line 2779 "upb/json/parser.rl"
{ end_subobject_full(parser); }
break;
#line 3229 "upb/json/parser.c"
#line 3234 "upb/json/parser.c"
}
}
}
@ -3233,7 +3238,7 @@ goto _again;} }
_out: {}
}
#line 2807 "upb/json/parser.rl"
#line 2812 "upb/json/parser.rl"
if (p != pe) {
upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p);
@ -3276,13 +3281,13 @@ static void json_parser_reset(upb_json_parser *p) {
/* Emit Ragel initialization of the parser. */
#line 3280 "upb/json/parser.c"
#line 3285 "upb/json/parser.c"
{
cs = json_start;
top = 0;
}
#line 2849 "upb/json/parser.rl"
#line 2854 "upb/json/parser.rl"
p->current_state = cs;
p->parser_top = top;
accumulate_clear(p);

@ -138,6 +138,21 @@ static TestCase kTestRoundtripMessagesPreserve[] = {
TEST_SENTINEL
};
static TestCase kTestSkipUnknown[] = {
{
TEST("{\"optionalEnum\":\"UNKNOWN_ENUM_VALUE\"}"),
EXPECT("{}"),
},
TEST_SENTINEL
};
static TestCase kTestFailure[] = {
{
TEST("{\"optionalEnum\":\"UNKNOWN_ENUM_VALUE\"}"),
},
TEST_SENTINEL
};
class StringSink {
public:
StringSink() {
@ -173,13 +188,15 @@ void test_json_roundtrip_message(const char* json_src,
const char* json_expected,
const upb::Handlers* serialize_handlers,
const upb::json::ParserMethodPtr parser_method,
int seam) {
int seam,
bool ignore_unknown) {
VerboseParserEnvironment env(verbose);
StringSink data_sink;
upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create(
env.arena(), serialize_handlers, data_sink.Sink());
upb::json::ParserPtr parser = upb::json::ParserPtr::Create(
env.arena(), parser_method, NULL, printer.input(), env.status(), false);
env.arena(), parser_method, NULL, printer.input(),
env.status(), ignore_unknown);
env.ResetBytesSink(parser.input());
env.Reset(json_src, strlen(json_src), false, false);
@ -196,8 +213,8 @@ void test_json_roundtrip_message(const char* json_src,
data_sink.Data().size())) {
fprintf(stderr,
"JSON parse/serialize roundtrip result differs:\n"
"Original:\n%s\nParsed/Serialized:\n%s\n",
json_src, data_sink.Data().c_str());
"Expected:\n%s\nParsed/Serialized:\n%s\n",
json_expected, data_sink.Data().c_str());
abort();
}
}
@ -225,7 +242,23 @@ void test_json_roundtrip() {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i);
serialize_handlers, parser_method, i,
false);
}
}
// Tests ignore unknown.
for (const TestCase* test_case = kTestSkipUnknown;
test_case->input != NULL; test_case++) {
const char *expected =
(test_case->expected == EXPECT_SAME) ?
test_case->input :
test_case->expected;
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i,
true);
}
}
@ -241,7 +274,52 @@ void test_json_roundtrip() {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i);
serialize_handlers, parser_method, i,
false);
}
}
}
void test_json_parse_failure(const char* json_src,
const upb::Handlers* serialize_handlers,
const upb::json::ParserMethodPtr parser_method,
int seam) {
VerboseParserEnvironment env(verbose);
StringSink data_sink;
upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create(
env.arena(), serialize_handlers, data_sink.Sink());
upb::json::ParserPtr parser = upb::json::ParserPtr::Create(
env.arena(), parser_method, NULL, printer.input(), env.status(), false);
env.ResetBytesSink(parser.input());
env.Reset(json_src, strlen(json_src), false, true);
bool ok = env.Start() &&
env.ParseBuffer(seam) &&
env.ParseBuffer(-1) &&
env.End();
ASSERT(!ok);
ASSERT(env.CheckConsistency());
}
// Starts with a proto message in JSON format, parses and expects failre.
void test_json_failure() {
upb::SymbolTable symtab;
upb::HandlerCache serialize_handlercache(
upb::json::PrinterPtr::NewCache(false));
upb::json::CodeCache parse_codecache;
upb::MessageDefPtr md(upb_test_json_TestMessage_getmsgdef(symtab.ptr()));
ASSERT(md);
const upb::Handlers* serialize_handlers = serialize_handlercache.Get(md);
const upb::json::ParserMethodPtr parser_method = parse_codecache.Get(md);
ASSERT(serialize_handlers);
for (const TestCase* test_case = kTestFailure;
test_case->input != NULL; test_case++) {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_parse_failure(test_case->input, serialize_handlers,
parser_method, i);
}
}
}
@ -251,6 +329,7 @@ int run_tests(int argc, char *argv[]) {
UPB_UNUSED(argc);
UPB_UNUSED(argv);
test_json_roundtrip();
test_json_failure();
return 0;
}
}

@ -1370,7 +1370,12 @@ static bool end_stringval_nontop(upb_json_parser *p) {
upb_selector_t sel = parser_getsel(p);
upb_sink_putint32(p->top->sink, sel, int_val);
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
if (p->ignore_json_unknown) {
ok = true;
/* TODO(teboring): Should also clean this field. */
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
}
}
break;

Loading…
Cancel
Save