Chiebot-Mirror
/
protobuf
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Ignore Unknown Enum Value (#280)

Browse Source 
* Add tests for proto2

* Fix ignore unknown enum value

* Fix CheckConsistency

* Add note to mention clear field for unknown enum value

* Update Comment

* Fix generated test

* Fix generated test
pull/13171/head
Paul Yang 5 years ago committed by GitHub
parent 6c4acba610
commit c26e8db53d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 166 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 145
      generated_for_cmake/upb/json/parser.c
  2. 91
      tests/json/test_json.cc
  3. 7
      upb/json/parser.rl

145
generated_for_cmake/upb/json/parser.c
Unescape View File

@ -1372,7 +1372,12 @@ static bool end_stringval_nontop(upb_json_parser *p) {
upb_selector_t sel = parser_getsel(p);
upb_sink_putint32(p->top->sink, sel, int_val);
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
if (p->ignore_json_unknown) {
ok = true;
/* TODO(teboring): Should also clean this field. */
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
}
}
break;
@ -2577,11 +2582,11 @@ static bool does_fieldmask_end(upb_json_parser *p) {
* final state once, when the closing '"' is seen. */
#line 2782 "upb/json/parser.rl"
#line 2787 "upb/json/parser.rl"
#line 2585 "upb/json/parser.c"
#line 2590 "upb/json/parser.c"
static const char _json_actions[] = {
0, 1, 0, 1, 1, 1, 3, 1,
4, 1, 6, 1, 7, 1, 8, 1,
@ -2836,7 +2841,7 @@ static const int json_en_value_machine = 78;
static const int json_en_main = 1;
#line 2785 "upb/json/parser.rl"
#line 2790 "upb/json/parser.rl"
size_t parse(void *closure, const void *hd, const char *buf, size_t size,
const upb_bufhandle *handle) {
@ -2859,7 +2864,7 @@ size_t parse(void *closure, const void *hd, const char *buf, size_t size,
capture_resume(parser, buf);
#line 2863 "upb/json/parser.c"
#line 2868 "upb/json/parser.c"
{
int _klen;
unsigned int _trans;
@ -2934,147 +2939,147 @@ _match:
switch ( *_acts++ )
{
case 1:
#line 2590 "upb/json/parser.rl"
#line 2595 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 2:
#line 2592 "upb/json/parser.rl"
#line 2597 "upb/json/parser.rl"
{ p--; {stack[top++] = cs; cs = 23;goto _again;} }
break;
case 3:
#line 2596 "upb/json/parser.rl"
#line 2601 "upb/json/parser.rl"
{ start_text(parser, p); }
break;
case 4:
#line 2597 "upb/json/parser.rl"
#line 2602 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_text(parser, p)); }
break;
case 5:
#line 2603 "upb/json/parser.rl"
#line 2608 "upb/json/parser.rl"
{ start_hex(parser); }
break;
case 6:
#line 2604 "upb/json/parser.rl"
#line 2609 "upb/json/parser.rl"
{ hexdigit(parser, p); }
break;
case 7:
#line 2605 "upb/json/parser.rl"
#line 2610 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_hex(parser)); }
break;
case 8:
#line 2611 "upb/json/parser.rl"
#line 2616 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(escape(parser, p)); }
break;
case 9:
#line 2617 "upb/json/parser.rl"
#line 2622 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 10:
#line 2622 "upb/json/parser.rl"
#line 2627 "upb/json/parser.rl"
{ start_year(parser, p); }
break;
case 11:
#line 2623 "upb/json/parser.rl"
#line 2628 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_year(parser, p)); }
break;
case 12:
#line 2627 "upb/json/parser.rl"
#line 2632 "upb/json/parser.rl"
{ start_month(parser, p); }
break;
case 13:
#line 2628 "upb/json/parser.rl"
#line 2633 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_month(parser, p)); }
break;
case 14:
#line 2632 "upb/json/parser.rl"
#line 2637 "upb/json/parser.rl"
{ start_day(parser, p); }
break;
case 15:
#line 2633 "upb/json/parser.rl"
#line 2638 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_day(parser, p)); }
break;
case 16:
#line 2637 "upb/json/parser.rl"
#line 2642 "upb/json/parser.rl"
{ start_hour(parser, p); }
break;
case 17:
#line 2638 "upb/json/parser.rl"
#line 2643 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_hour(parser, p)); }
break;
case 18:
#line 2642 "upb/json/parser.rl"
#line 2647 "upb/json/parser.rl"
{ start_minute(parser, p); }
break;
case 19:
#line 2643 "upb/json/parser.rl"
#line 2648 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_minute(parser, p)); }
break;
case 20:
#line 2647 "upb/json/parser.rl"
#line 2652 "upb/json/parser.rl"
{ start_second(parser, p); }
break;
case 21:
#line 2648 "upb/json/parser.rl"
#line 2653 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_second(parser, p)); }
break;
case 22:
#line 2653 "upb/json/parser.rl"
#line 2658 "upb/json/parser.rl"
{ start_duration_base(parser, p); }
break;
case 23:
#line 2654 "upb/json/parser.rl"
#line 2659 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_duration_base(parser, p)); }
break;
case 24:
#line 2656 "upb/json/parser.rl"
#line 2661 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 25:
#line 2661 "upb/json/parser.rl"
#line 2666 "upb/json/parser.rl"
{ start_timestamp_base(parser); }
break;
case 26:
#line 2663 "upb/json/parser.rl"
#line 2668 "upb/json/parser.rl"
{ start_timestamp_fraction(parser, p); }
break;
case 27:
#line 2664 "upb/json/parser.rl"
#line 2669 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_timestamp_fraction(parser, p)); }
break;
case 28:
#line 2666 "upb/json/parser.rl"
#line 2671 "upb/json/parser.rl"
{ start_timestamp_zone(parser, p); }
break;
case 29:
#line 2667 "upb/json/parser.rl"
#line 2672 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_timestamp_zone(parser, p)); }
break;
case 30:
#line 2669 "upb/json/parser.rl"
#line 2674 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 31:
#line 2674 "upb/json/parser.rl"
#line 2679 "upb/json/parser.rl"
{ start_fieldmask_path_text(parser, p); }
break;
case 32:
#line 2675 "upb/json/parser.rl"
#line 2680 "upb/json/parser.rl"
{ end_fieldmask_path_text(parser, p); }
break;
case 33:
#line 2680 "upb/json/parser.rl"
#line 2685 "upb/json/parser.rl"
{ start_fieldmask_path(parser); }
break;
case 34:
#line 2681 "upb/json/parser.rl"
#line 2686 "upb/json/parser.rl"
{ end_fieldmask_path(parser); }
break;
case 35:
#line 2687 "upb/json/parser.rl"
#line 2692 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
case 36:
#line 2692 "upb/json/parser.rl"
#line 2697 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) {
{stack[top++] = cs; cs = 47;goto _again;}
@ -3088,11 +3093,11 @@ _match:
}
break;
case 37:
#line 2705 "upb/json/parser.rl"
#line 2710 "upb/json/parser.rl"
{ p--; {stack[top++] = cs; cs = 78;goto _again;} }
break;
case 38:
#line 2710 "upb/json/parser.rl"
#line 2715 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
start_any_member(parser, p);
@ -3102,11 +3107,11 @@ _match:
}
break;
case 39:
#line 2717 "upb/json/parser.rl"
#line 2722 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_membername(parser)); }
break;
case 40:
#line 2720 "upb/json/parser.rl"
#line 2725 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
end_any_member(parser, p);
@ -3116,7 +3121,7 @@ _match:
}
break;
case 41:
#line 2731 "upb/json/parser.rl"
#line 2736 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
start_any_object(parser, p);
@ -3126,7 +3131,7 @@ _match:
}
break;
case 42:
#line 2740 "upb/json/parser.rl"
#line 2745 "upb/json/parser.rl"
{
if (is_wellknown_msg(parser, UPB_WELLKNOWN_ANY)) {
CHECK_RETURN_TOP(end_any_object(parser, p));
@ -3136,54 +3141,54 @@ _match:
}
break;
case 43:
#line 2752 "upb/json/parser.rl"
#line 2757 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_array(parser)); }
break;
case 44:
#line 2756 "upb/json/parser.rl"
#line 2761 "upb/json/parser.rl"
{ end_array(parser); }
break;
case 45:
#line 2761 "upb/json/parser.rl"
#line 2766 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_number(parser, p)); }
break;
case 46:
#line 2762 "upb/json/parser.rl"
#line 2767 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_number(parser, p)); }
break;
case 47:
#line 2764 "upb/json/parser.rl"
#line 2769 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_stringval(parser)); }
break;
case 48:
#line 2765 "upb/json/parser.rl"
#line 2770 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_stringval(parser)); }
break;
case 49:
#line 2767 "upb/json/parser.rl"
#line 2772 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, true)); }
break;
case 50:
#line 2769 "upb/json/parser.rl"
#line 2774 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, false)); }
break;
case 51:
#line 2771 "upb/json/parser.rl"
#line 2776 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_null(parser)); }
break;
case 52:
#line 2773 "upb/json/parser.rl"
#line 2778 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(start_subobject_full(parser)); }
break;
case 53:
#line 2774 "upb/json/parser.rl"
#line 2779 "upb/json/parser.rl"
{ end_subobject_full(parser); }
break;
case 54:
#line 2779 "upb/json/parser.rl"
#line 2784 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; goto _again;} }
break;
#line 3187 "upb/json/parser.c"
#line 3192 "upb/json/parser.c"
}
}
@ -3200,32 +3205,32 @@ _again:
while ( __nacts-- > 0 ) {
switch ( *__acts++ ) {
case 0:
#line 2588 "upb/json/parser.rl"
#line 2593 "upb/json/parser.rl"
{ p--; {cs = stack[--top]; if ( p == pe )
goto _test_eof;
goto _again;} }
break;
case 46:
#line 2762 "upb/json/parser.rl"
#line 2767 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_number(parser, p)); }
break;
case 49:
#line 2767 "upb/json/parser.rl"
#line 2772 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, true)); }
break;
case 50:
#line 2769 "upb/json/parser.rl"
#line 2774 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_bool(parser, false)); }
break;
case 51:
#line 2771 "upb/json/parser.rl"
#line 2776 "upb/json/parser.rl"
{ CHECK_RETURN_TOP(end_null(parser)); }
break;
case 53:
#line 2774 "upb/json/parser.rl"
#line 2779 "upb/json/parser.rl"
{ end_subobject_full(parser); }
break;
#line 3229 "upb/json/parser.c"
#line 3234 "upb/json/parser.c"
}
}
}
@ -3233,7 +3238,7 @@ goto _again;} }
_out: {}
}
#line 2807 "upb/json/parser.rl"
#line 2812 "upb/json/parser.rl"
if (p != pe) {
upb_status_seterrf(parser->status, "Parse error at '%.*s'\n", pe - p, p);
@ -3276,13 +3281,13 @@ static void json_parser_reset(upb_json_parser *p) {
/* Emit Ragel initialization of the parser. */
#line 3280 "upb/json/parser.c"
#line 3285 "upb/json/parser.c"
{
cs = json_start;
top = 0;
}
#line 2849 "upb/json/parser.rl"
#line 2854 "upb/json/parser.rl"
p->current_state = cs;
p->parser_top = top;
accumulate_clear(p);

91
tests/json/test_json.cc
Unescape View File

@ -138,6 +138,21 @@ static TestCase kTestRoundtripMessagesPreserve[] = {
TEST_SENTINEL
};
static TestCase kTestSkipUnknown[] = {
{
TEST("{\"optionalEnum\":\"UNKNOWN_ENUM_VALUE\"}"),
EXPECT("{}"),
},
TEST_SENTINEL
};
static TestCase kTestFailure[] = {
{
TEST("{\"optionalEnum\":\"UNKNOWN_ENUM_VALUE\"}"),
},
TEST_SENTINEL
};
class StringSink {
public:
StringSink() {
@ -173,13 +188,15 @@ void test_json_roundtrip_message(const char* json_src,
const char* json_expected,
const upb::Handlers* serialize_handlers,
const upb::json::ParserMethodPtr parser_method,
int seam) {
int seam,
bool ignore_unknown) {
VerboseParserEnvironment env(verbose);
StringSink data_sink;
upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create(
env.arena(), serialize_handlers, data_sink.Sink());
upb::json::ParserPtr parser = upb::json::ParserPtr::Create(
env.arena(), parser_method, NULL, printer.input(), env.status(), false);
env.arena(), parser_method, NULL, printer.input(),
env.status(), ignore_unknown);
env.ResetBytesSink(parser.input());
env.Reset(json_src, strlen(json_src), false, false);
@ -196,8 +213,8 @@ void test_json_roundtrip_message(const char* json_src,
data_sink.Data().size())) {
fprintf(stderr,
"JSON parse/serialize roundtrip result differs:\n"
"Original:\n%s\nParsed/Serialized:\n%s\n",
json_src, data_sink.Data().c_str());
"Expected:\n%s\nParsed/Serialized:\n%s\n",
json_expected, data_sink.Data().c_str());
abort();
}
}
@ -225,7 +242,23 @@ void test_json_roundtrip() {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i);
serialize_handlers, parser_method, i,
false);
}
}
// Tests ignore unknown.
for (const TestCase* test_case = kTestSkipUnknown;
test_case->input != NULL; test_case++) {
const char *expected =
(test_case->expected == EXPECT_SAME) ?
test_case->input :
test_case->expected;
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i,
true);
}
}
@ -241,7 +274,52 @@ void test_json_roundtrip() {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_roundtrip_message(test_case->input, expected,
serialize_handlers, parser_method, i);
serialize_handlers, parser_method, i,
false);
}
}
}
void test_json_parse_failure(const char* json_src,
const upb::Handlers* serialize_handlers,
const upb::json::ParserMethodPtr parser_method,
int seam) {
VerboseParserEnvironment env(verbose);
StringSink data_sink;
upb::json::PrinterPtr printer = upb::json::PrinterPtr::Create(
env.arena(), serialize_handlers, data_sink.Sink());
upb::json::ParserPtr parser = upb::json::ParserPtr::Create(
env.arena(), parser_method, NULL, printer.input(), env.status(), false);
env.ResetBytesSink(parser.input());
env.Reset(json_src, strlen(json_src), false, true);
bool ok = env.Start() &&
env.ParseBuffer(seam) &&
env.ParseBuffer(-1) &&
env.End();
ASSERT(!ok);
ASSERT(env.CheckConsistency());
}
// Starts with a proto message in JSON format, parses and expects failre.
void test_json_failure() {
upb::SymbolTable symtab;
upb::HandlerCache serialize_handlercache(
upb::json::PrinterPtr::NewCache(false));
upb::json::CodeCache parse_codecache;
upb::MessageDefPtr md(upb_test_json_TestMessage_getmsgdef(symtab.ptr()));
ASSERT(md);
const upb::Handlers* serialize_handlers = serialize_handlercache.Get(md);
const upb::json::ParserMethodPtr parser_method = parse_codecache.Get(md);
ASSERT(serialize_handlers);
for (const TestCase* test_case = kTestFailure;
test_case->input != NULL; test_case++) {
for (size_t i = 0; i < strlen(test_case->input); i++) {
test_json_parse_failure(test_case->input, serialize_handlers,
parser_method, i);
}
}
}
@ -251,6 +329,7 @@ int run_tests(int argc, char *argv[]) {
UPB_UNUSED(argc);
UPB_UNUSED(argv);
test_json_roundtrip();
test_json_failure();
return 0;
}
}

7
upb/json/parser.rl
Unescape View File

@ -1370,7 +1370,12 @@ static bool end_stringval_nontop(upb_json_parser *p) {
upb_selector_t sel = parser_getsel(p);
upb_sink_putint32(p->top->sink, sel, int_val);
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
if (p->ignore_json_unknown) {
ok = true;
/* TODO(teboring): Should also clean this field. */
} else {
upb_status_seterrf(p->status, "Enum value unknown: '%.*s'", len, buf);
}
}
break;

Write Preview
Loading…
Cancel
Save