From 78a73333b705a9d184da7f3174533500d05b2a21 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Jun 2016 08:24:44 -0700 Subject: [PATCH 01/10] Fix memory leak upon receiving two RST_STREAMs --- .../chttp2/transport/frame_rst_stream.c | 14 +- .../2c452818a10ddef09b90c89a53db14b9b56b21f3 | Bin 0 -> 52 bytes .../42ead79c94eccdf8a8c3d8036be73e14fa260dd5 | Bin 0 -> 64 bytes .../4e05d6cf1c3f0c04f6ee92d09a53ee0fe35c085a | Bin 0 -> 64 bytes .../8f980dd25f1c77e3536131c2c620aa32e8c13180 | Bin 0 -> 14 bytes .../aef36c49d7dec0dcf8cdc224d9e9221fa2cb1db0 | Bin 0 -> 53 bytes ...h-14ed70cd9ea7987cdd0c8f6e39398ee7c60ee2ff | Bin 0 -> 719 bytes .../dcb06a6e34cbed15515e5b3581ca666f704777bd | Bin 0 -> 238 bytes .../ea46b684f1e67a27c231f2d536c41da631189b9c | Bin 0 -> 696 bytes tools/run_tests/tests.json | 152 ++++++++++++++++++ 10 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/2c452818a10ddef09b90c89a53db14b9b56b21f3 create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/42ead79c94eccdf8a8c3d8036be73e14fa260dd5 create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/4e05d6cf1c3f0c04f6ee92d09a53ee0fe35c085a create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/8f980dd25f1c77e3536131c2c620aa32e8c13180 create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/aef36c49d7dec0dcf8cdc224d9e9221fa2cb1db0 create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/crash-14ed70cd9ea7987cdd0c8f6e39398ee7c60ee2ff create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/dcb06a6e34cbed15515e5b3581ca666f704777bd create mode 100644 test/core/end2end/fuzzers/client_fuzzer_corpus/ea46b684f1e67a27c231f2d536c41da631189b9c diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index a7aefb99158..e3a3c9e4a7c 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -102,12 +102,14 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse( if (p->byte == 4) { GPR_ASSERT(is_last); stream_parsing->received_close = 1; - stream_parsing->forced_close_error = grpc_error_set_int( - GRPC_ERROR_CREATE("RST_STREAM"), GRPC_ERROR_INT_HTTP2_ERROR, - (intptr_t)((((uint32_t)p->reason_bytes[0]) << 24) | - (((uint32_t)p->reason_bytes[1]) << 16) | - (((uint32_t)p->reason_bytes[2]) << 8) | - (((uint32_t)p->reason_bytes[3])))); + if (stream_parsing->forced_close_error == GRPC_ERROR_NONE) { + stream_parsing->forced_close_error = grpc_error_set_int( + GRPC_ERROR_CREATE("RST_STREAM"), GRPC_ERROR_INT_HTTP2_ERROR, + (intptr_t)((((uint32_t)p->reason_bytes[0]) << 24) | + (((uint32_t)p->reason_bytes[1]) << 16) | + (((uint32_t)p->reason_bytes[2]) << 8) | + (((uint32_t)p->reason_bytes[3])))); + } } return GRPC_ERROR_NONE; diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/2c452818a10ddef09b90c89a53db14b9b56b21f3 b/test/core/end2end/fuzzers/client_fuzzer_corpus/2c452818a10ddef09b90c89a53db14b9b56b21f3 new file mode 100644 index 0000000000000000000000000000000000000000..059634fda10a679b7bb4f753fe06660888b26dd4 GIT binary patch literal 52 zcmZS3WMpJ#`1cCr>)SNic&kq1P C_Z2e$ literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/42ead79c94eccdf8a8c3d8036be73e14fa260dd5 b/test/core/end2end/fuzzers/client_fuzzer_corpus/42ead79c94eccdf8a8c3d8036be73e14fa260dd5 new file mode 100644 index 0000000000000000000000000000000000000000..b9c53b26edd94105bc4bdb24e73697cc4be93dc4 GIT binary patch literal 64 zcmZQz&|+j^U|?Wm;7KnkNY*XM%uUTNE#U@pic1npN{a=Jfbu+y3``8!3=GU5r40Z7 LGqQs;X{`VNxT6g} literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/4e05d6cf1c3f0c04f6ee92d09a53ee0fe35c085a b/test/core/end2end/fuzzers/client_fuzzer_corpus/4e05d6cf1c3f0c04f6ee92d09a53ee0fe35c085a new file mode 100644 index 0000000000000000000000000000000000000000..8a4a279998d72b2aa9b94310364f40c2580515c2 GIT binary patch literal 64 zcmZQz&|+j^U|?Wm;7KnkNY*XM%uUTNE#X!Naf(Y4OG=9cjeznzj0{W+*$fQKAf*id M|1+|KG-<5>0Is79{{R30 literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/8f980dd25f1c77e3536131c2c620aa32e8c13180 b/test/core/end2end/fuzzers/client_fuzzer_corpus/8f980dd25f1c77e3536131c2c620aa32e8c13180 new file mode 100644 index 0000000000000000000000000000000000000000..fcebab7a64f25b8dfad2cbb19851b76833a674b7 GIT binary patch literal 14 TcmZS3WMpJ#`1cCKLz? literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/aef36c49d7dec0dcf8cdc224d9e9221fa2cb1db0 b/test/core/end2end/fuzzers/client_fuzzer_corpus/aef36c49d7dec0dcf8cdc224d9e9221fa2cb1db0 new file mode 100644 index 0000000000000000000000000000000000000000..6b015fe66e6131d746ff41da4b961b0ea074a02f GIT binary patch literal 53 wcmZQzU||3OMg|5&AjJ%3v*hM7Fz|zz42+Dqxr(`e{sRfWsS`IUP6RU40etxhaR2}S literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/crash-14ed70cd9ea7987cdd0c8f6e39398ee7c60ee2ff b/test/core/end2end/fuzzers/client_fuzzer_corpus/crash-14ed70cd9ea7987cdd0c8f6e39398ee7c60ee2ff new file mode 100644 index 0000000000000000000000000000000000000000..be6366049d93b585d80d625c72ec8eabc458f8f5 GIT binary patch literal 719 zcma))!A%1(5Jl~fL&S-#01X$%d*ezZL|E_oKS9FwrjM^T*-qe*OC#i~h0zgCtcA1q!ShX>2v)Qh6$Z`_$uwt&j-F4ZR; z5Sv7_I-eYEmYW?vcl;IP1o{i~ad|8#&gc-zlgX7X`Jb@uP`1h)s-Wx}23A)%p@>wE za!C@8^?ad=qK}-dq9bit8{GOE-Go^{jRIXY;!;t6Li0vv6=jI|D`%R^2vry~vi7F? zj%DqTNjKogO;hm+{BgT~2z2|($`EkI19~LJC`TB;F;RQr8{}99X|={K^4w4OLpkCM RBFep{S=%Njy`zq%`UA}C*FFFM literal 0 HcmV?d00001 diff --git a/test/core/end2end/fuzzers/client_fuzzer_corpus/dcb06a6e34cbed15515e5b3581ca666f704777bd b/test/core/end2end/fuzzers/client_fuzzer_corpus/dcb06a6e34cbed15515e5b3581ca666f704777bd new file mode 100644 index 0000000000000000000000000000000000000000..92750f94a32152bdd71d388f47c4bcefb6ec34b8 GIT binary patch literal 238 zcmY+7F%AMj2t~o2!q`|@8)C7Jm9en!5Zu87aF}11*jjo!PvH2O)odig1Rng4NKyI5 zR^;7UGq%I&coxyJ30Y55)dt5*_uO3%k6!p!;!iRS_{tfJY8zgY`Q<}Y|ly-<3XI{!F o4-v0C#t0){Z=6m|-N-+eun0W93o~cbV+vGj(5A~7+}Qfx0kNN|?f?J) literal 0 HcmV?d00001 diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 3ed7a6bc476..2353ac85831 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -54188,6 +54188,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/2c452818a10ddef09b90c89a53db14b9b56b21f3" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/2c6e69067c68c145dc5d3a60b86d8081fdf95d0d" @@ -54986,6 +55005,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/42ead79c94eccdf8a8c3d8036be73e14fa260dd5" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/43202ad9b1a689d919ab9ae91c2d0223394867bf" @@ -55328,6 +55366,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/4e05d6cf1c3f0c04f6ee92d09a53ee0fe35c085a" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/4e21c4b5c454df51c102f09ea1ba78c42133ee16" @@ -57247,6 +57304,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/8f980dd25f1c77e3536131c2c620aa32e8c13180" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/90a9c3390752b94ca19a58cb2fe6267bc818f718" @@ -58463,6 +58539,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/aef36c49d7dec0dcf8cdc224d9e9221fa2cb1db0" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/af8b24ffaecdfaf96c0cd7c76f31dc9e1b4d0935" @@ -59508,6 +59603,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/crash-14ed70cd9ea7987cdd0c8f6e39398ee7c60ee2ff" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/crash-3bd02c98286bfa7be8e13c5500ddb587bba74fbb" @@ -60173,6 +60287,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/dcb06a6e34cbed15515e5b3581ca666f704777bd" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/dccd1fd6d3394f5f68c87950ed7356a2e9ef0f6f" @@ -60667,6 +60800,25 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/ea46b684f1e67a27c231f2d536c41da631189b9c" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/eb969b9ab1b0d6b5d197795223ba7a091ebd8460" From 56565fca7532fcc559cd8149e437eb0a7ecdfd9b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Jun 2016 08:36:53 -0700 Subject: [PATCH 02/10] Force receiving a SETTINGS frame as the first frame --- .../ext/transport/chttp2/transport/chttp2_transport.c | 3 ++- src/core/ext/transport/chttp2/transport/internal.h | 1 + src/core/ext/transport/chttp2/transport/parsing.c | 11 +++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9aa39ba26c2..a8c4b271fe1 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -264,6 +264,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->parsing.is_client = is_client; t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; + t->parsing.is_first_frame = true; t->writing.is_client = is_client; t->optional_drop_message = gpr_empty_slice(); grpc_connectivity_state_init( @@ -1805,7 +1806,7 @@ static void post_reading_action_locked(grpc_exec_ctx *exec_ctx, UNREF_TRANSPORT(exec_ctx, t, "reading_action"); } - GRPC_ERROR_UNREF(error); + GRPC_LOG_IF_ERROR("close_transport", error); } /******************************************************************************* diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 54e72ebd249..64385b43724 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -265,6 +265,7 @@ struct grpc_chttp2_transport_parsing { uint8_t incoming_frame_type; uint8_t incoming_frame_flags; uint8_t header_eof; + bool is_first_frame; uint32_t expect_continuation_stream_id; uint32_t incoming_frame_size; uint32_t incoming_stream_id; diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 785134091f5..991d7729af4 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -468,6 +468,17 @@ grpc_error *grpc_chttp2_perform_read( static grpc_error *init_frame_parser( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { + if (transport_parsing->is_first_frame && + transport_parsing->incoming_frame_type != GRPC_CHTTP2_FRAME_SETTINGS) { + char *msg; + gpr_asprintf( + &msg, "Expected SETTINGS frame as the first frame, got frame type %d", + transport_parsing->incoming_frame_type); + grpc_error *err = GRPC_ERROR_CREATE(msg); + gpr_free(msg); + return err; + } + transport_parsing->is_first_frame = false; if (transport_parsing->expect_continuation_stream_id != 0) { if (transport_parsing->incoming_frame_type != GRPC_CHTTP2_FRAME_CONTINUATION) { From daec5f94d2805db094cd748dbb691e25185af3c6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Jun 2016 09:24:45 -0700 Subject: [PATCH 03/10] We dont need to account for parsing or not when writing --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 +-- src/core/ext/transport/chttp2/transport/internal.h | 3 +-- src/core/ext/transport/chttp2/transport/writing.c | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index a8c4b271fe1..4eb5ba8270e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -644,8 +644,7 @@ static void finish_global_actions(grpc_exec_ctx *exec_ctx, for (;;) { if (!t->executor.writing_active && !t->closed && - grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing, - t->executor.parsing_active)) { + grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing)) { t->executor.writing_active = 1; REF_TRANSPORT(t, "writing"); prevent_endpoint_shutdown(t); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 64385b43724..909c406f8a8 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -525,8 +525,7 @@ struct grpc_chttp2_stream { are required, and schedule them if so */ int grpc_chttp2_unlocking_check_writes(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *global, - grpc_chttp2_transport_writing *writing, - int is_parsing); + grpc_chttp2_transport_writing *writing); void grpc_chttp2_perform_writes( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, grpc_endpoint *endpoint); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index add76781827..b19f5f068df 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -45,7 +45,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, int grpc_chttp2_unlocking_check_writes( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing, int is_parsing) { + grpc_chttp2_transport_writing *transport_writing) { grpc_chttp2_stream_global *stream_global; grpc_chttp2_stream_writing *stream_writing; @@ -61,7 +61,7 @@ int grpc_chttp2_unlocking_check_writes( [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); if (transport_global->dirtied_local_settings && - !transport_global->sent_local_settings && !is_parsing) { + !transport_global->sent_local_settings) { gpr_slice_buffer_add( &transport_writing->outbuf, grpc_chttp2_settings_create( From fa1f74e2267d8dfe97d617357033d2a0bf8423be Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Jun 2016 11:11:44 -0700 Subject: [PATCH 04/10] Fix errored streams prematurely terminating, add a test --- Makefile | 2 + .../chttp2/transport/chttp2_transport.c | 4 +- .../ext/transport/chttp2/transport/internal.h | 1 + test/core/end2end/end2end_nosec_tests.c | 8 + test/core/end2end/end2end_tests.c | 8 + test/core/end2end/gen_build_yaml.py | 1 + .../end2end/tests/streaming_error_response.c | 278 ++++++++ tools/run_tests/sources_and_headers.json | 2 + tools/run_tests/tests.json | 629 +++++++++++++++++- .../end2end_nosec_tests.vcxproj | 2 + .../end2end_nosec_tests.vcxproj.filters | 3 + .../tests/end2end_tests/end2end_tests.vcxproj | 2 + .../end2end_tests.vcxproj.filters | 3 + 13 files changed, 932 insertions(+), 11 deletions(-) create mode 100644 test/core/end2end/tests/streaming_error_response.c diff --git a/Makefile b/Makefile index 13bc2f1871c..23e40cfb075 100644 --- a/Makefile +++ b/Makefile @@ -6389,6 +6389,7 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/tests/simple_delayed_request.c \ test/core/end2end/tests/simple_metadata.c \ test/core/end2end/tests/simple_request.c \ + test/core/end2end/tests/streaming_error_response.c \ test/core/end2end/tests/trailing_metadata.c \ PUBLIC_HEADERS_C += \ @@ -6465,6 +6466,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/tests/simple_delayed_request.c \ test/core/end2end/tests/simple_metadata.c \ test/core/end2end/tests/simple_request.c \ + test/core/end2end/tests/streaming_error_response.c \ test/core/end2end/tests/trailing_metadata.c \ PUBLIC_HEADERS_C += \ diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9aa39ba26c2..63d07108a7c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1093,6 +1093,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, stream_global->recv_trailing_metadata_finished = add_closure_barrier(on_complete); stream_global->recv_trailing_metadata = op->recv_trailing_metadata; + stream_global->final_metadata_requested = true; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } @@ -1246,7 +1247,8 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx, stream_global->recv_initial_metadata_ready = NULL; } if (stream_global->recv_message_ready != NULL) { - while (stream_global->seen_error && + while (stream_global->final_metadata_requested && + stream_global->seen_error && (bs = grpc_chttp2_incoming_frame_queue_pop( &stream_global->incoming_frames)) != NULL) { incoming_byte_stream_destroy_locked(exec_ctx, NULL, NULL, bs); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 54e72ebd249..48602fb47ce 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -440,6 +440,7 @@ typedef struct { bool published_initial_metadata; bool published_trailing_metadata; + bool final_metadata_requested; grpc_chttp2_incoming_metadata_buffer received_initial_metadata; grpc_chttp2_incoming_metadata_buffer received_trailing_metadata; diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index b71299c09e0..2893bddc43c 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -115,6 +115,8 @@ extern void simple_metadata(grpc_end2end_test_config config); extern void simple_metadata_pre_init(void); extern void simple_request(grpc_end2end_test_config config); extern void simple_request_pre_init(void); +extern void streaming_error_response(grpc_end2end_test_config config); +extern void streaming_error_response_pre_init(void); extern void trailing_metadata(grpc_end2end_test_config config); extern void trailing_metadata_pre_init(void); @@ -157,6 +159,7 @@ void grpc_end2end_tests_pre_init(void) { simple_delayed_request_pre_init(); simple_metadata_pre_init(); simple_request_pre_init(); + streaming_error_response_pre_init(); trailing_metadata_pre_init(); } @@ -203,6 +206,7 @@ void grpc_end2end_tests(int argc, char **argv, simple_delayed_request(config); simple_metadata(config); simple_request(config); + streaming_error_response(config); trailing_metadata(config); return; } @@ -352,6 +356,10 @@ void grpc_end2end_tests(int argc, char **argv, simple_request(config); continue; } + if (0 == strcmp("streaming_error_response", argv[i])) { + streaming_error_response(config); + continue; + } if (0 == strcmp("trailing_metadata", argv[i])) { trailing_metadata(config); continue; diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c index 00c9c44a78c..96a38e76dcc 100644 --- a/test/core/end2end/end2end_tests.c +++ b/test/core/end2end/end2end_tests.c @@ -117,6 +117,8 @@ extern void simple_metadata(grpc_end2end_test_config config); extern void simple_metadata_pre_init(void); extern void simple_request(grpc_end2end_test_config config); extern void simple_request_pre_init(void); +extern void streaming_error_response(grpc_end2end_test_config config); +extern void streaming_error_response_pre_init(void); extern void trailing_metadata(grpc_end2end_test_config config); extern void trailing_metadata_pre_init(void); @@ -160,6 +162,7 @@ void grpc_end2end_tests_pre_init(void) { simple_delayed_request_pre_init(); simple_metadata_pre_init(); simple_request_pre_init(); + streaming_error_response_pre_init(); trailing_metadata_pre_init(); } @@ -207,6 +210,7 @@ void grpc_end2end_tests(int argc, char **argv, simple_delayed_request(config); simple_metadata(config); simple_request(config); + streaming_error_response(config); trailing_metadata(config); return; } @@ -360,6 +364,10 @@ void grpc_end2end_tests(int argc, char **argv, simple_request(config); continue; } + if (0 == strcmp("streaming_error_response", argv[i])) { + streaming_error_response(config); + continue; + } if (0 == strcmp("trailing_metadata", argv[i])) { trailing_metadata(config); continue; diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 325d9b3cad0..6d3d8f8d3c4 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -126,6 +126,7 @@ END2END_TESTS = { 'simple_delayed_request': connectivity_test_options, 'simple_metadata': default_test_options, 'simple_request': default_test_options, + 'streaming_error_response': default_test_options, 'trailing_metadata': default_test_options, } diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c new file mode 100644 index 00000000000..c1d6f73ecdc --- /dev/null +++ b/test/core/end2end/tests/streaming_error_response.c @@ -0,0 +1,278 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/core/end2end/end2end_tests.h" + +#include +#include + +#include +#include +#include +#include +#include +#include "test/core/end2end/cq_verifier.h" + +enum { TIMEOUT = 200000 }; + +static void *tag(intptr_t t) { return (void *)t; } + +static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, + const char *test_name, + grpc_channel_args *client_args, + grpc_channel_args *server_args, + bool request_status_early) { + grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s/%s/request_status_early=%s", test_name, config.name, + request_status_early ? "true" : "false"); + f = config.create_fixture(client_args, server_args); + config.init_server(&f, server_args); + config.init_client(&f, client_args); + return f; +} + +static gpr_timespec n_seconds_time(int n) { + return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n); +} + +static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } + +static void drain_cq(grpc_completion_queue *cq) { + grpc_event ev; + do { + ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + } while (ev.type != GRPC_QUEUE_SHUTDOWN); +} + +static void shutdown_server(grpc_end2end_test_fixture *f) { + if (!f->server) return; + grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); + GPR_ASSERT(grpc_completion_queue_pluck( + f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL) + .type == GRPC_OP_COMPLETE); + grpc_server_destroy(f->server); + f->server = NULL; +} + +static void shutdown_client(grpc_end2end_test_fixture *f) { + if (!f->client) return; + grpc_channel_destroy(f->client); + f->client = NULL; +} + +static void end_test(grpc_end2end_test_fixture *f) { + shutdown_server(f); + shutdown_client(f); + + grpc_completion_queue_shutdown(f->cq); + drain_cq(f->cq); + grpc_completion_queue_destroy(f->cq); +} + +/* Client sends a request with payload, server reads then returns status. */ +static void test(grpc_end2end_test_config config, bool request_status_early) { + grpc_call *c; + grpc_call *s; + gpr_slice response_payload1_slice = gpr_slice_from_copied_string("hello"); + grpc_byte_buffer *response_payload1 = + grpc_raw_byte_buffer_create(&response_payload1_slice, 1); + gpr_slice response_payload2_slice = gpr_slice_from_copied_string("world"); + grpc_byte_buffer *response_payload2 = + grpc_raw_byte_buffer_create(&response_payload2_slice, 1); + gpr_timespec deadline = five_seconds_time(); + grpc_end2end_test_fixture f = begin_test(config, "streaming_error_response", + NULL, NULL, request_status_early); + cq_verifier *cqv = cq_verifier_create(f.cq); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer *response_payload1_recv = NULL; + grpc_byte_buffer *response_payload2_recv = NULL; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + char *details = NULL; + size_t details_capacity = 0; + int was_cancelled = 2; + + c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, + "/foo", "foo.test.google.fr", deadline, NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata = &initial_metadata_recv; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = &response_payload1_recv; + op++; + if (request_status_early) { + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->data.recv_status_on_client.status_details_capacity = &details_capacity; + op++; + } + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call( + f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101))); + cq_expect_completion(cqv, tag(101), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message = response_payload1; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + cq_expect_completion(cqv, tag(102), 1); + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message = response_payload2; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(103), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + cq_expect_completion(cqv, tag(103), 1); + if (!request_status_early) { + cq_expect_completion(cqv, tag(1), 1); + } + cq_verify(cqv); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_FAILED_PRECONDITION; + op->data.send_status_from_server.status_details = "xyz"; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(104), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + if (!request_status_early) { + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = &response_payload2_recv; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(2), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + } + + cq_expect_completion(cqv, tag(104), 1); + if (request_status_early) { + cq_expect_completion(cqv, tag(1), 1); + } else { + cq_expect_completion(cqv, tag(2), 1); + } + cq_verify(cqv); + + if (!request_status_early) { + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->data.recv_status_on_client.status_details_capacity = &details_capacity; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(3), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + cq_expect_completion(cqv, tag(3), 1); + cq_verify(cqv); + + GPR_ASSERT(response_payload1_recv != NULL); + GPR_ASSERT(response_payload2_recv != NULL); + } + + GPR_ASSERT(status == GRPC_STATUS_FAILED_PRECONDITION); + GPR_ASSERT(0 == strcmp(details, "xyz")); + GPR_ASSERT(0 == strcmp(call_details.method, "/foo")); + GPR_ASSERT(0 == strcmp(call_details.host, "foo.test.google.fr")); + GPR_ASSERT(was_cancelled == 1); + + gpr_free(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_destroy(c); + grpc_call_destroy(s); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(response_payload1); + grpc_byte_buffer_destroy(response_payload2); + grpc_byte_buffer_destroy(response_payload1_recv); + grpc_byte_buffer_destroy(response_payload2_recv); + + end_test(&f); + config.tear_down_data(&f); +} + +void streaming_error_response(grpc_end2end_test_config config) { + test(config, false); + test(config, true); +} + +void streaming_error_response_pre_init(void) {} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 57e3bd63f42..750d98b0be0 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -5400,6 +5400,7 @@ "test/core/end2end/tests/simple_delayed_request.c", "test/core/end2end/tests/simple_metadata.c", "test/core/end2end/tests/simple_request.c", + "test/core/end2end/tests/streaming_error_response.c", "test/core/end2end/tests/trailing_metadata.c" ], "third_party": false, @@ -5458,6 +5459,7 @@ "test/core/end2end/tests/simple_delayed_request.c", "test/core/end2end/tests/simple_metadata.c", "test/core/end2end/tests/simple_request.c", + "test/core/end2end/tests/streaming_error_response.c", "test/core/end2end/tests/trailing_metadata.c" ], "third_party": false, diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 3ed7a6bc476..f0e4849ec99 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -5186,6 +5186,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -6022,6 +6044,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -6821,6 +6865,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -7482,6 +7547,26 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fd_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -8316,6 +8401,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -8930,6 +9037,22 @@ "linux" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, { "args": [ "trailing_metadata" @@ -9738,6 +9861,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -10574,6 +10719,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_loadreporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -11373,6 +11540,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -12045,6 +12233,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -12738,6 +12947,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -13410,6 +13640,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -14105,7 +14356,7 @@ }, { "args": [ - "trailing_metadata" + "streaming_error_response" ], "ci_platforms": [ "windows", @@ -14126,19 +14377,18 @@ }, { "args": [ - "bad_hostname" + "trailing_metadata" ], "ci_platforms": [ "windows", "linux", - "mac", "posix" ], "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, "language": "c", - "name": "h2_ssl_test", + "name": "h2_sockpair_1byte_test", "platforms": [ "windows", "linux", @@ -14148,7 +14398,7 @@ }, { "args": [ - "binary_metadata" + "bad_hostname" ], "ci_platforms": [ "windows", @@ -14170,7 +14420,7 @@ }, { "args": [ - "call_creds" + "binary_metadata" ], "ci_platforms": [ "windows", @@ -14192,7 +14442,7 @@ }, { "args": [ - "cancel_after_accept" + "call_creds" ], "ci_platforms": [ "windows", @@ -14200,7 +14450,7 @@ "mac", "posix" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, "language": "c", @@ -14214,7 +14464,7 @@ }, { "args": [ - "cancel_after_client_done" + "cancel_after_accept" ], "ci_platforms": [ "windows", @@ -14222,7 +14472,29 @@ "mac", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "cancel_after_client_done" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, "exclude_configs": [], "flaky": false, "language": "c", @@ -14938,6 +15210,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -15774,6 +16068,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_cert_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -16447,6 +16763,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -17188,6 +17525,26 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -18000,6 +18357,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -18814,6 +19193,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -19456,6 +19857,26 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -20268,6 +20689,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -20866,6 +21309,22 @@ "linux" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, { "args": [ "trailing_metadata" @@ -21652,6 +22111,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -22466,6 +22947,28 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_loadreporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -23118,6 +23621,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -23790,6 +24314,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -24441,6 +24986,27 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -25175,6 +25741,29 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [ + "msan" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" @@ -25898,6 +26487,26 @@ "posix" ] }, + { + "args": [ + "streaming_error_response" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "trailing_metadata" diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj index 22cd102d116..923c1d1ab41 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj @@ -225,6 +225,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters index 1bb208bba8e..6533eaa057b 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters @@ -112,6 +112,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj index bfd437e8717..0b859e25ce4 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj @@ -227,6 +227,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters index 61c065f77ca..ea1c5e3c237 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters @@ -115,6 +115,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests From 89dde5e5681a7a718b234311e302897f5861723c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Jun 2016 11:13:11 -0700 Subject: [PATCH 05/10] Fix copyright --- test/core/end2end/tests/streaming_error_response.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c index c1d6f73ecdc..e15c132d633 100644 --- a/test/core/end2end/tests/streaming_error_response.c +++ b/test/core/end2end/tests/streaming_error_response.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From ffaafe6fa3fbb308c00619bef45e9f43a3d8e4ba Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 20 Jun 2016 19:36:49 +0000 Subject: [PATCH 06/10] Change with_call from parameter to attribute --- src/python/grpcio/grpc/__init__.py | 57 +++++++++++++++---- src/python/grpcio/grpc/_channel.py | 32 ++++++++--- .../grpcio/grpc/beta/_client_adaptations.py | 8 +-- .../grpcio/tests/unit/_metadata_test.py | 8 +-- src/python/grpcio/tests/unit/_rpc_test.py | 21 +++---- 5 files changed, 87 insertions(+), 39 deletions(-) diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 28adca37723..f9b09b4cf14 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -438,9 +438,7 @@ class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)): """Affords invoking a unary-unary RPC.""" @abc.abstractmethod - def __call__( - self, request, timeout=None, metadata=None, credentials=None, - with_call=False): + def __call__(self, request, timeout=None, metadata=None, credentials=None): """Synchronously invokes the underlying RPC. Args: @@ -449,12 +447,30 @@ class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)): metadata: An optional sequence of pairs of bytes to be transmitted to the service-side of the RPC. credentials: An optional CallCredentials for the RPC. - with_call: Whether or not to include return a Call for the RPC in addition - to the response. Returns: - The response value for the RPC, and a Call for the RPC if with_call was - set to True at invocation. + The response value for the RPC. + + Raises: + RpcError: Indicating that the RPC terminated with non-OK status. The + raised RpcError will also be a Call for the RPC affording the RPC's + metadata, status code, and details. + """ + raise NotImplementedError() + + @abc.abstractmethod + def with_call(self, request, timeout=None, metadata=None, credentials=None): + """Synchronously invokes the underlying RPC. + + Args: + request: The request value for the RPC. + timeout: An optional durating of time in seconds to allow for the RPC. + metadata: An optional sequence of pairs of bytes to be transmitted to the + service-side of the RPC. + credentials: An optional CallCredentials for the RPC. + + Returns: + The response value for the RPC and a Call value for the RPC. Raises: RpcError: Indicating that the RPC terminated with non-OK status. The @@ -510,8 +526,7 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)): @abc.abstractmethod def __call__( - self, request_iterator, timeout=None, metadata=None, credentials=None, - with_call=False): + self, request_iterator, timeout=None, metadata=None, credentials=None): """Synchronously invokes the underlying RPC. Args: @@ -520,8 +535,6 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)): metadata: An optional sequence of pairs of bytes to be transmitted to the service-side of the RPC. credentials: An optional CallCredentials for the RPC. - with_call: Whether or not to include return a Call for the RPC in addition - to the response. Returns: The response value for the RPC, and a Call for the RPC if with_call was @@ -534,6 +547,28 @@ class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)): """ raise NotImplementedError() + @abc.abstractmethod + def with_call( + self, request_iterator, timeout=None, metadata=None, credentials=None): + """Synchronously invokes the underlying RPC. + + Args: + request_iterator: An iterator that yields request values for the RPC. + timeout: An optional duration of time in seconds to allow for the RPC. + metadata: An optional sequence of pairs of bytes to be transmitted to the + service-side of the RPC. + credentials: An optional CallCredentials for the RPC. + + Returns: + The response value for the RPC and a Call for the RPC. + + Raises: + RpcError: Indicating that the RPC terminated with non-OK status. The + raised RpcError will also be a Call for the RPC affording the RPC's + metadata, status code, and details. + """ + raise NotImplementedError() + @abc.abstractmethod def future( self, request_iterator, timeout=None, metadata=None, credentials=None): diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index d9315d2e6cd..8b774c44488 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -449,9 +449,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): ) return state, operations, deadline, deadline_timespec, None - def __call__( - self, request, timeout=None, metadata=None, credentials=None, - with_call=False): + def _blocking(self, request, timeout, metadata, credentials): state, operations, deadline, deadline_timespec, rendezvous = self._prepare( request, timeout, metadata) if rendezvous: @@ -464,7 +462,15 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): call.set_credentials(credentials._credentials) call.start_batch(cygrpc.Operations(operations), None) _handle_event(completion_queue.poll(), state, self._response_deserializer) - return _end_unary_response_blocking(state, with_call, deadline) + return state, deadline + + def __call__(self, request, timeout=None, metadata=None, credentials=None): + state, deadline, = self._blocking(request, timeout, metadata, credentials) + return _end_unary_response_blocking(state, False, deadline) + + def with_call(self, request, timeout=None, metadata=None, credentials=None): + state, deadline, = self._blocking(request, timeout, metadata, credentials) + return _end_unary_response_blocking(state, True, deadline) def future(self, request, timeout=None, metadata=None, credentials=None): state, operations, deadline, deadline_timespec, rendezvous = self._prepare( @@ -532,9 +538,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): self._request_serializer = request_serializer self._response_deserializer = response_deserializer - def __call__( - self, request_iterator, timeout=None, metadata=None, credentials=None, - with_call=False): + def _blocking(self, request_iterator, timeout, metadata, credentials): deadline, deadline_timespec = _deadline(timeout) state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None) completion_queue = cygrpc.CompletionQueue() @@ -563,7 +567,19 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): state.condition.notify_all() if not state.due: break - return _end_unary_response_blocking(state, with_call, deadline) + return state, deadline + + def __call__( + self, request_iterator, timeout=None, metadata=None, credentials=None): + state, deadline, = self._blocking( + request_iterator, timeout, metadata, credentials) + return _end_unary_response_blocking(state, False, deadline) + + def with_call( + self, request_iterator, timeout=None, metadata=None, credentials=None): + state, deadline, = self._blocking( + request_iterator, timeout, metadata, credentials) + return _end_unary_response_blocking(state, True, deadline) def future( self, request_iterator, timeout=None, metadata=None, credentials=None): diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index 024808c5404..56456cc117f 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -186,9 +186,9 @@ def _blocking_unary_unary( response_deserializer=response_deserializer) effective_metadata = _effective_metadata(metadata, metadata_transformer) if with_call: - response, call = multi_callable( + response, call = multi_callable.with_call( request, timeout=timeout, metadata=effective_metadata, - credentials=_credentials(protocol_options), with_call=True) + credentials=_credentials(protocol_options)) return response, _Rendezvous(None, None, call) else: return multi_callable( @@ -237,9 +237,9 @@ def _blocking_stream_unary( response_deserializer=response_deserializer) effective_metadata = _effective_metadata(metadata, metadata_transformer) if with_call: - response, call = multi_callable( + response, call = multi_callable.with_call( request_iterator, timeout=timeout, metadata=effective_metadata, - credentials=_credentials(protocol_options), with_call=True) + credentials=_credentials(protocol_options)) return response, _Rendezvous(None, None, call) else: return multi_callable( diff --git a/src/python/grpcio/tests/unit/_metadata_test.py b/src/python/grpcio/tests/unit/_metadata_test.py index 77b39012619..2cb13f236bd 100644 --- a/src/python/grpcio/tests/unit/_metadata_test.py +++ b/src/python/grpcio/tests/unit/_metadata_test.py @@ -173,8 +173,8 @@ class MetadataTest(unittest.TestCase): def testUnaryUnary(self): multi_callable = self._channel.unary_unary(_UNARY_UNARY) - unused_response, call = multi_callable( - _REQUEST, metadata=_CLIENT_METADATA, with_call=True) + unused_response, call = multi_callable.with_call( + _REQUEST, metadata=_CLIENT_METADATA) self.assertTrue(test_common.metadata_transmitted( _SERVER_INITIAL_METADATA, call.initial_metadata())) self.assertTrue(test_common.metadata_transmitted( @@ -192,9 +192,9 @@ class MetadataTest(unittest.TestCase): def testStreamUnary(self): multi_callable = self._channel.stream_unary(_STREAM_UNARY) - unused_response, call = multi_callable( + unused_response, call = multi_callable.with_call( [_REQUEST] * test_constants.STREAM_LENGTH, - metadata=_CLIENT_METADATA, with_call=True) + metadata=_CLIENT_METADATA) self.assertTrue(test_common.metadata_transmitted( _SERVER_INITIAL_METADATA, call.initial_metadata())) self.assertTrue(test_common.metadata_transmitted( diff --git a/src/python/grpcio/tests/unit/_rpc_test.py b/src/python/grpcio/tests/unit/_rpc_test.py index 8407593c86d..9814504edff 100644 --- a/src/python/grpcio/tests/unit/_rpc_test.py +++ b/src/python/grpcio/tests/unit/_rpc_test.py @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Test of gRPC Python's application-layer API.""" +"""Test of RPCs made against gRPC Python's application-layer API.""" import itertools import threading @@ -216,10 +216,9 @@ class RPCTest(unittest.TestCase): expected_response = self._handler.handle_unary_unary(request, None) multi_callable = _unary_unary_multi_callable(self._channel) - response, call = multi_callable( + response, call = multi_callable.with_call( request, metadata=( - (b'test', b'SuccessfulUnaryRequestBlockingUnaryResponseWithCall'),), - with_call=True) + (b'test', b'SuccessfulUnaryRequestBlockingUnaryResponseWithCall'),)) self.assertEqual(expected_response, response) self.assertIs(grpc.StatusCode.OK, call.code()) @@ -266,11 +265,11 @@ class RPCTest(unittest.TestCase): request_iterator = iter(requests) multi_callable = _stream_unary_multi_callable(self._channel) - response, call = multi_callable( + response, call = multi_callable.with_call( request_iterator, metadata=( (b'test', b'SuccessfulStreamRequestBlockingUnaryResponseWithCall'), - ), with_call=True) + )) self.assertEqual(expected_response, response) self.assertIs(grpc.StatusCode.OK, call.code()) @@ -525,10 +524,9 @@ class RPCTest(unittest.TestCase): multi_callable = _unary_unary_multi_callable(self._channel) with self._control.pause(): with self.assertRaises(grpc.RpcError) as exception_context: - multi_callable( + multi_callable.with_call( request, timeout=test_constants.SHORT_TIMEOUT, - metadata=((b'test', b'ExpiredUnaryRequestBlockingUnaryResponse'),), - with_call=True) + metadata=((b'test', b'ExpiredUnaryRequestBlockingUnaryResponse'),)) self.assertIsNotNone(exception_context.exception.initial_metadata()) self.assertIs( @@ -640,10 +638,9 @@ class RPCTest(unittest.TestCase): multi_callable = _unary_unary_multi_callable(self._channel) with self._control.fail(): with self.assertRaises(grpc.RpcError) as exception_context: - multi_callable( + multi_callable.with_call( request, - metadata=((b'test', b'FailedUnaryRequestBlockingUnaryResponse'),), - with_call=True) + metadata=((b'test', b'FailedUnaryRequestBlockingUnaryResponse'),)) self.assertIs(grpc.StatusCode.UNKNOWN, exception_context.exception.code()) From 3b5ae53cd149ecb821c8950b19b06c8330e288c0 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 20 Jun 2016 22:09:39 +0000 Subject: [PATCH 07/10] Correct lingering FATAL_FAILURE sites This should have been done as part of 5444bed32f1405ebb53b0c37d3b. --- src/python/grpcio/tests/unit/_channel_connectivity_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/tests/unit/_channel_connectivity_test.py b/src/python/grpcio/tests/unit/_channel_connectivity_test.py index a1575efada7..ae8de523ecf 100644 --- a/src/python/grpcio/tests/unit/_channel_connectivity_test.py +++ b/src/python/grpcio/tests/unit/_channel_connectivity_test.py @@ -135,12 +135,12 @@ class ChannelConnectivityTest(unittest.TestCase): self.assertNotIn( grpc.ChannelConnectivity.TRANSIENT_FAILURE, third_connectivities) self.assertNotIn( - grpc.ChannelConnectivity.FATAL_FAILURE, third_connectivities) + grpc.ChannelConnectivity.SHUTDOWN, third_connectivities) self.assertNotIn( grpc.ChannelConnectivity.TRANSIENT_FAILURE, fourth_connectivities) self.assertNotIn( - grpc.ChannelConnectivity.FATAL_FAILURE, fourth_connectivities) + grpc.ChannelConnectivity.SHUTDOWN, fourth_connectivities) def test_reachable_then_unreachable_channel_connectivity(self): server = _server.Server((), futures.ThreadPoolExecutor(max_workers=0)) From 74ea91d55229a654d0987d2f39cddf8514926f14 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 20 Jun 2016 18:19:41 -0700 Subject: [PATCH 08/10] use qps_json_driver for latency profiling --- .../latency_profile/run_latency_profile.sh | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index 54a25a9cb73..40c6fcb4314 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,17 +28,61 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# format argument via +# $ echo '{...}' | python -mjson.tool +read -r -d '' SCENARIOS_JSON_ARG <<'EOF' +{ + "scenarios": [ + { + "benchmark_seconds": 5, + "client_config": { + "client_channels": 1, + "client_type": "SYNC_CLIENT", + "histogram_params": { + "max_possible": 60000000000.0, + "resolution": 0.01 + }, + "load_params": { + "closed_loop": {} + }, + "outstanding_rpcs_per_channel": 1, + "payload_config": { + "simple_params": { + "req_size": 0, + "resp_size": 0 + } + }, + "rpc_type": "UNARY", + "security_params": { + "server_host_override": "foo.test.google.fr", + "use_test_ca": true + } + }, + "name": "cpp_protobuf_sync_unary_ping_pong_secure", + "num_clients": 1, + "num_servers": 1, + "server_config": { + "core_limit": 1, + "security_params": { + "server_host_override": "foo.test.google.fr", + "use_test_ca": true + }, + "server_type": "SYNC_SERVER" + }, + "spawn_local_worker_count": 2, + "warmup_seconds": 5 + } + ] +} + +EOF + set -ex cd $(dirname $0)/../../.. -BINS="sync_unary_ping_pong_test sync_streaming_ping_pong_test" CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` -make CONFIG=basicprof -j$CPUS $BINS - -mkdir -p reports - # try to use pypy for generating reports # each trace dumps 7-8gig of text to disk, and processing this into a report is # heavyweight - so any speed boost is worthwhile @@ -49,35 +93,13 @@ else PYTHON=python2.7 fi -# start processes, interleaving report index generation +make CONFIG=basicprof -j$CPUS qps_json_driver + +mkdir -p reports echo '' > reports/index.html -for bin in $BINS -do - bins/basicprof/$bin - mv latency_trace.txt $bin.trace - echo "$bin
" >> reports/index.html -done -pids="" -# generate report pages... this will take some time -# run them in parallel: they take 1 cpu each -for bin in $BINS -do - $PYTHON tools/profiling/latency_profile/profile_analyzer.py \ - --source=$bin.trace --fmt=simple > reports/$bin.txt & - pids+=" $!" -done +bins/basicprof/qps_json_driver --scenarios_json="$SCENARIOS_JSON_ARG" +echo '
' >> reports/index.html
+$PYTHON tools/profiling/latency_profile/profile_analyzer.py \
+    --source=latency_trace.txt --fmt=simple >> reports/index.html
+echo '
' >> reports/index.html echo '' >> reports/index.html - -# make sure we kill the report generation if something goes wrong -trap "kill $pids || true" 0 - -# finally, wait for the background report generation to finish -for pid in $pids -do - if wait $pid - then - echo "Finished $pid" - else - exit 1 - fi -done From f8bbe72a00ef3827879ea8d377c595c8e96d6a66 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 21 Jun 2016 14:05:16 -0700 Subject: [PATCH 09/10] Make src/node/tools/package.json consistent with its template --- templates/src/node/tools/package.json.template | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/src/node/tools/package.json.template b/templates/src/node/tools/package.json.template index 69ad71a3b83..02824259767 100644 --- a/templates/src/node/tools/package.json.template +++ b/templates/src/node/tools/package.json.template @@ -36,6 +36,7 @@ "index.js", "bin/protoc.js", "bin/protoc_plugin.js", + "bin/google/protobuf", "LICENSE" ], "main": "index.js" From 9d393a5d3c1a2609c3aa44caa3879e590f659c21 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 21 Jun 2016 15:20:19 -0700 Subject: [PATCH 10/10] slightly fancier output --- tools/profiling/latency_profile/run_latency_profile.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/profiling/latency_profile/run_latency_profile.sh b/tools/profiling/latency_profile/run_latency_profile.sh index 40c6fcb4314..618db202dc4 100755 --- a/tools/profiling/latency_profile/run_latency_profile.sh +++ b/tools/profiling/latency_profile/run_latency_profile.sh @@ -96,10 +96,11 @@ fi make CONFIG=basicprof -j$CPUS qps_json_driver mkdir -p reports -echo '' > reports/index.html bins/basicprof/qps_json_driver --scenarios_json="$SCENARIOS_JSON_ARG" -echo '
' >> reports/index.html
+
+echo 'Latency profile for:
' > reports/index.html
+echo "
${SCENARIOS_JSON_ARG}
" >> reports/index.html
+echo '
' >> reports/index.html
 $PYTHON tools/profiling/latency_profile/profile_analyzer.py \
     --source=latency_trace.txt --fmt=simple >> reports/index.html
-echo '
' >> reports/index.html
-echo '' >> reports/index.html
+echo '

' >> reports/index.html