From 2426163b314d9dc7f841b433aed55fe3c6f17350 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 8 Sep 2017 16:12:05 -0700 Subject: [PATCH 01/23] Switch to three threads per cq for unconstrained benchmarks --- tools/run_tests/generated/tests.json | 64 +++++++++---------- .../run_tests/performance/scenario_config.py | 2 + 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ad39f2d6d06..1f97309a97a 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -49905,7 +49905,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -49955,7 +49955,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50005,7 +50005,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50105,7 +50105,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50205,7 +50205,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50255,7 +50255,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50305,7 +50305,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50355,7 +50355,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50759,7 +50759,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50809,7 +50809,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50859,7 +50859,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50959,7 +50959,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51059,7 +51059,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51109,7 +51109,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51159,7 +51159,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51209,7 +51209,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51884,7 +51884,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51960,7 +51960,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52036,7 +52036,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52188,7 +52188,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52340,7 +52340,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52416,7 +52416,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52492,7 +52492,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52568,7 +52568,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53180,7 +53180,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53256,7 +53256,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53332,7 +53332,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53484,7 +53484,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53636,7 +53636,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53712,7 +53712,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53788,7 +53788,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53864,7 +53864,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 86236b19bf3..b97650de67e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -452,6 +452,8 @@ class CXXLanguage: unconstrained_client=synchronicity, secure=secure, minimal_stack=not secure, + server_threads_per_cq=3, + client_threads_per_cq=3, categories=smoketest_categories+[SCALABLE]) # TODO(vjpai): Re-enable this test. It has a lot of timeouts From 41388890860f29cd6e147bde163f57322a408a2f Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 11 Sep 2017 22:47:38 +0000 Subject: [PATCH 02/23] Ignore all Python 3 virtual environments How soon could Python 4 happen anyway? :-P --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5e38f5fa01b..5ccad2e4f2f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ htmlcov/ dist/ *.egg py27/ -py34/ +py3[0-9]*/ # Node installation output node_modules From 4b7ef4d52546e05a1b33aac1eeb218409c4b0a3a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 23:09:22 -0700 Subject: [PATCH 03/23] Build a wall and make 'u' pay for it --- src/core/lib/iomgr/ev_epoll1_linux.c | 143 +++++++++++++-------------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de0..4efd705fa81 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -160,18 +160,18 @@ struct grpc_pollset_worker { (worker)->kick_state_mutator = __LINE__; \ } while (false) -#define MAX_NEIGHBOURHOODS 1024 +#define MAX_NEIGHBORHOODS 1024 -typedef struct pollset_neighbourhood { +typedef struct pollset_neighborhood { gpr_mu mu; grpc_pollset *active_root; char pad[GPR_CACHELINE_SIZE]; -} pollset_neighbourhood; +} pollset_neighborhood; struct grpc_pollset { gpr_mu mu; - pollset_neighbourhood *neighbourhood; - bool reassigning_neighbourhood; + pollset_neighborhood *neighborhood; + bool reassigning_neighborhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; @@ -384,8 +384,8 @@ GPR_TLS_DECL(g_current_thread_worker); /* The designated poller */ static gpr_atm g_active_poller; -static pollset_neighbourhood *g_neighbourhoods; -static size_t g_num_neighbourhoods; +static pollset_neighborhood *g_neighborhoods; +static size_t g_num_neighborhoods; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -424,8 +424,8 @@ static worker_remove_result worker_remove(grpc_pollset *pollset, } } -static size_t choose_neighbourhood(void) { - return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +static size_t choose_neighborhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighborhoods; } static grpc_error *pollset_global_init(void) { @@ -441,11 +441,11 @@ static grpc_error *pollset_global_init(void) { &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } - g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); - g_neighbourhoods = (pollset_neighbourhood *)gpr_zalloc( - sizeof(*g_neighbourhoods) * g_num_neighbourhoods); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_init(&g_neighbourhoods[i].mu); + g_num_neighborhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBORHOODS); + g_neighborhoods = (pollset_neighborhood *)gpr_zalloc( + sizeof(*g_neighborhoods) * g_num_neighborhoods); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_init(&g_neighborhoods[i].mu); } return GRPC_ERROR_NONE; } @@ -454,17 +454,17 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_destroy(&g_neighbourhoods[i].mu); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_destroy(&g_neighborhoods[i].mu); } - gpr_free(g_neighbourhoods); + gpr_free(g_neighborhoods); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; - pollset->reassigning_neighbourhood = false; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; + pollset->reassigning_neighborhood = false; pollset->root_worker = NULL; pollset->kicked_without_poller = false; pollset->seen_inactive = true; @@ -477,26 +477,26 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - if (pollset->neighbourhood != neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (pollset->neighborhood != neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } pollset->prev->next = pollset->next; pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = + if (pollset == pollset->neighborhood->active_root) { + pollset->neighborhood->active_root = pollset->next == pollset ? NULL : pollset->next; } } - gpr_mu_unlock(&pollset->neighbourhood->mu); + gpr_mu_unlock(&pollset->neighborhood->mu); } gpr_mu_unlock(&pollset->mu); gpr_mu_destroy(&pollset->mu); @@ -675,16 +675,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, // pollset has been observed to be inactive, we need to move back to the // active list bool is_reassigning = false; - if (!pollset->reassigning_neighbourhood) { + if (!pollset->reassigning_neighborhood) { is_reassigning = true; - pollset->reassigning_neighbourhood = true; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + pollset->reassigning_neighborhood = true; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; } - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", @@ -692,17 +692,17 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, is_reassigning); } if (pollset->seen_inactive) { - if (neighbourhood != pollset->neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (neighborhood != pollset->neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } /* In the brief time we released the pollset locks above, the worker MAY have been kicked. In this case, the worker should get out of this pollset ASAP and hence this should neither add the pollset to - neighbourhood nor mark the pollset as active. + neighborhood nor mark the pollset as active. On a side note, the only way a worker's kick state could have changed at this point is if it were "kicked specifically". Since the worker has @@ -710,25 +710,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, not visible in the "kick any" path yet */ if (worker->kick_state == UNKICKED) { pollset->seen_inactive = false; - if (neighbourhood->active_root == NULL) { - neighbourhood->active_root = pollset->next = pollset->prev = pollset; + if (neighborhood->active_root == NULL) { + neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ if (worker->kick_state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } } else { - pollset->next = neighbourhood->active_root; + pollset->next = neighborhood->active_root; pollset->prev = pollset->next->prev; pollset->next->prev = pollset->prev->next = pollset; } } } if (is_reassigning) { - GPR_ASSERT(pollset->reassigning_neighbourhood); - pollset->reassigning_neighbourhood = false; + GPR_ASSERT(pollset->reassigning_neighborhood); + pollset->reassigning_neighborhood = false; } - gpr_mu_unlock(&neighbourhood->mu); + gpr_mu_unlock(&neighborhood->mu); } worker_insert(pollset, worker); @@ -763,7 +763,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } /* We release pollset lock in this function at a couple of places: - * 1. Briefly when assigning pollset to a neighbourhood + * 1. Briefly when assigning pollset to a neighborhood * 2. When doing gpr_cv_wait() * It is possible that 'kicked_without_poller' was set to true during (1) and * 'shutting_down' is set to true during (1) or (2). If either of them is @@ -781,12 +781,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; } -static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { - GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); +static bool check_neighborhood_for_available_poller( + pollset_neighborhood *neighborhood) { + GPR_TIMER_BEGIN("check_neighborhood_for_available_poller", 0); bool found_worker = false; do { - grpc_pollset *inspect = neighbourhood->active_root; + grpc_pollset *inspect = neighborhood->active_root; if (inspect == NULL) { break; } @@ -831,8 +831,8 @@ static bool check_neighbourhood_for_available_poller( gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect); } inspect->seen_inactive = true; - if (inspect == neighbourhood->active_root) { - neighbourhood->active_root = + if (inspect == neighborhood->active_root) { + neighborhood->active_root = inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; @@ -841,7 +841,7 @@ static bool check_neighbourhood_for_available_poller( } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - GPR_TIMER_END("check_neighbourhood_for_available_poller", 0); + GPR_TIMER_END("check_neighborhood_for_available_poller", 0); return found_worker; } @@ -873,32 +873,31 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); - size_t poller_neighbourhood_idx = - (size_t)(pollset->neighbourhood - g_neighbourhoods); + size_t poller_neighborhood_idx = + (size_t)(pollset->neighborhood - g_neighborhoods); gpr_mu_unlock(&pollset->mu); bool found_worker = false; - bool scan_state[MAX_NEIGHBOURHOODS]; - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - if (gpr_mu_trylock(&neighbourhood->mu)) { - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + bool scan_state[MAX_NEIGHBORHOODS]; + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + if (gpr_mu_trylock(&neighborhood->mu)) { + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); scan_state[i] = true; } else { scan_state[i] = false; } } - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { if (scan_state[i]) continue; - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + gpr_mu_lock(&neighborhood->mu); + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); From f11a40d23b982129a56fc5e167ebc213afe52e6d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: [PATCH 04/23] Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ .../run_tests/performance/bq_upload_result.py | 2 + .../performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++--- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af2..f665b5808e6 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac231962..31819d6159c 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 00000000000..1ed5be731de --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 00000000000..c5d884344ff --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7f..a28d27fece2 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file From 3a24364b48dcce2f7d84a1c8e8b755cf8925e997 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 12 Sep 2017 12:06:27 -0700 Subject: [PATCH 05/23] Fix a memory leak in cares wrapper --- .../client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 0d71f3560e0..2e2b411ab8d 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -174,7 +174,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET6_ADDRSTRLEN]; ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN); @@ -195,7 +195,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET_ADDRSTRLEN]; ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN); From 5d6156033e2fff01abd6b01b49626fed95a6d81e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: [PATCH 06/23] Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- .../performance/massage_qps_stats.py | 118 ++++++++++-------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 +++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e6..985c1f51c9b 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731de..30a94ca7cdf 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344ff..400a0c82bfd 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece2..84b3cb7c0a3 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", From 73a61708965d35e3b96d7a4f8feeba85601d5aa2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: [PATCH 07/23] Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ .../run_tests/performance/bq_upload_result.py | 2 + .../performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++--- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af2..f665b5808e6 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac231962..31819d6159c 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 00000000000..1ed5be731de --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 00000000000..c5d884344ff --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7f..a28d27fece2 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file From 27147b85f8eeede4f918705e42a91093d450757c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: [PATCH 08/23] Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- .../performance/massage_qps_stats.py | 118 ++++++++++-------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 +++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e6..985c1f51c9b 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731de..30a94ca7cdf 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344ff..400a0c82bfd 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece2..84b3cb7c0a3 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", From 52778c4729ae4e59eadae3bb49728967f397eb6f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:00:11 -0700 Subject: [PATCH 09/23] Adding pointer conversions. Renaming a few variables and type names to avoid C++ compilation issues --- src/core/lib/channel/channel_args.c | 4 +- src/core/lib/channel/channel_stack_builder.c | 16 ++--- src/core/lib/compression/stream_compression.c | 3 +- src/core/lib/iomgr/combiner.c | 3 +- src/core/lib/iomgr/error.c | 34 +++++------ src/core/lib/iomgr/ev_epoll1_linux.c | 6 +- src/core/lib/iomgr/ev_epollex_linux.c | 45 +++++++------- src/core/lib/iomgr/polling_entity.h | 4 +- src/core/lib/iomgr/resource_quota.c | 6 +- src/core/lib/iomgr/socket_factory_posix.c | 4 +- src/core/lib/iomgr/socket_mutator.c | 4 +- src/core/lib/iomgr/timer_manager.c | 2 +- src/core/lib/iomgr/udp_server.c | 2 +- src/core/lib/slice/slice.c | 4 +- src/core/lib/surface/call.c | 58 ++++++++++--------- 15 files changed, 101 insertions(+), 94 deletions(-) diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 16d0737b405..9215707f1c3 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -212,7 +212,7 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a) { grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, a->args[i].key)) { @@ -226,7 +226,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( grpc_stream_compression_algorithm grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_STREAM_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index df45f13c813..b663ebfb522 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -214,13 +214,13 @@ bool grpc_channel_stack_builder_prepend_filter( static void add_after(filter_node *before, const grpc_channel_filter *filter, grpc_post_filter_create_init_func post_init_func, void *user_data) { - filter_node *new = (filter_node *)gpr_malloc(sizeof(*new)); - new->next = before->next; - new->prev = before; - new->next->prev = new->prev->next = new; - new->filter = filter; - new->init = post_init_func; - new->init_arg = user_data; + filter_node *new_node = (filter_node *)gpr_malloc(sizeof(*new_node)); + new_node->next = before->next; + new_node->prev = before; + new_node->next->prev = new_node->prev->next = new_node; + new_node->filter = filter; + new_node->init = post_init_func; + new_node->init_arg = user_data; } bool grpc_channel_stack_builder_add_filter_before( @@ -268,7 +268,7 @@ grpc_error *grpc_channel_stack_builder_finish( // create an array of filters const grpc_channel_filter **filters = - gpr_malloc(sizeof(*filters) * num_filters); + (const grpc_channel_filter **)gpr_malloc(sizeof(*filters) * num_filters); size_t i = 0; for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) { filters[i++] = p->filter; diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index df13d53e06f..ba9302794f7 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -159,7 +159,8 @@ bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_stream_compression_context *grpc_stream_compression_context_create( grpc_stream_compression_method method) { grpc_stream_compression_context *ctx = - gpr_zalloc(sizeof(grpc_stream_compression_context)); + (grpc_stream_compression_context *)gpr_zalloc( + sizeof(grpc_stream_compression_context)); int r; if (ctx == NULL) { return NULL; diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 360967f3ba2..f899b25f105 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -356,7 +356,8 @@ static void combiner_finally_exec(grpc_exec_ctx *exec_ctx, static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure, grpc_error *error) { - combiner_finally_exec(exec_ctx, closure, GRPC_ERROR_REF(error)); + combiner_finally_exec(exec_ctx, (grpc_closure *)closure, + GRPC_ERROR_REF(error)); } grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner) { diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index dcd175a2e1a..d5e00a8f643 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -278,13 +278,13 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, memcpy((*err)->arena + slot, &value, sizeof(value)); } -static void internal_add_error(grpc_error **err, grpc_error *new) { - grpc_linked_error new_last = {new, UINT8_MAX}; +static void internal_add_error(grpc_error **err, grpc_error *new_err) { + grpc_linked_error new_last = {new_err, UINT8_MAX}; uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); if (slot == UINT8_MAX) { - gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new, - grpc_error_string(new)); - GRPC_ERROR_UNREF(new); + gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, + new_err, grpc_error_string(new_err)); + GRPC_ERROR_UNREF(new_err); return; } if ((*err)->first_err == UINT8_MAX) { @@ -321,8 +321,8 @@ grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc, uint8_t initial_arena_capacity = (uint8_t)( DEFAULT_ERROR_CAPACITY + (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY); - grpc_error *err = - gpr_malloc(sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); + grpc_error *err = (grpc_error *)gpr_malloc( + sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); if (err == NULL) { // TODO(ctiller): make gpr_malloc return NULL return GRPC_ERROR_OOM; } @@ -432,10 +432,10 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which, intptr_t value) { GPR_TIMER_BEGIN("grpc_error_set_int", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_int(&new, which, value); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_int(&new_err, which, value); GPR_TIMER_END("grpc_error_set_int", 0); - return new; + return new_err; } typedef struct { @@ -477,10 +477,10 @@ bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) { grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, grpc_slice str) { GPR_TIMER_BEGIN("grpc_error_set_str", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_str(&new, which, str); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_str(&new_err, which, str); GPR_TIMER_END("grpc_error_set_str", 0); - return new; + return new_err; } bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, @@ -507,10 +507,10 @@ bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) { GPR_TIMER_BEGIN("grpc_error_add_child", 0); - grpc_error *new = copy_error_and_unref(src); - internal_add_error(&new, child); + grpc_error *new_err = copy_error_and_unref(src); + internal_add_error(&new_err, child); GPR_TIMER_END("grpc_error_add_child", 0); - return new; + return new_err; } static const char *no_error_string = "\"No Error\""; @@ -733,7 +733,7 @@ const char *grpc_error_string(grpc_error *err) { void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string); if (p != NULL) { GPR_TIMER_END("grpc_error_string", 0); - return p; + return (const char *)p; } kv_pairs kvs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de0..01d28220c71 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; -static const char *kick_state_string(kick_state st) { +static const char *kick_state_string(kick_state_t st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state st) { } struct grpc_pollset_worker { - kick_state kick_state; + kick_state_t kick_state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 277347ac704..df69025f1a3 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable { +typedef struct pollable_t { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable; +} pollable_t; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return ""; } -static char *pollable_desc(pollable *p) { +static char *pollable_desc(pollable_t *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable *p) { return out; } -static pollable g_empty_pollable; +static pollable_t g_empty_pollable; -static void pollable_init(pollable *p, polling_obj_type type); -static void pollable_destroy(pollable *p); +static void pollable_init(pollable_t *p, polling_obj_type type); +static void pollable_destroy(pollable_t *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p); +static grpc_error *pollable_materialize(pollable_t *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable pollable; + pollable_t pollable; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable *pollable; + pollable_t *pollable; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable pollable; - pollable *current_pollable; + pollable_t pollable; + pollable_t *current_pollable; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable *p, polling_obj_type type) { +static void pollable_init(pollable_t *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable *p) { +static void pollable_destroy(pollable_t *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p) { +static grpc_error *pollable_materialize(pollable_t *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -599,7 +599,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,7 +664,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; + pollable_t *p = pollset->current_pollable; if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } @@ -744,7 +744,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { return p != &g_empty_pollable && p != &pollset->pollable; } @@ -762,8 +762,9 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p got pollset_wakeup %p", pollset, data_ptr); } - append_error(&error, grpc_wakeup_fd_consume_wakeup( - (void *)((~(intptr_t)1) & (intptr_t)data_ptr)), + append_error(&error, + grpc_wakeup_fd_consume_wakeup( + (grpc_wakeup_fd *)((~(intptr_t)1) & (intptr_t)data_ptr)), err_desc); } else { grpc_fd *fd = (grpc_fd *)data_ptr; @@ -800,7 +801,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable *p, gpr_timespec now, + pollable_t *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -1028,7 +1029,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable */ + /* empty pollable --> single fd pollable_t */ pollset_kick_all(exec_ctx, pollset); pollset->current_pollable = &fd->pollable; if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 971fd88b428..940f10f152d 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,6 +22,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" +typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; + /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an * abstract interface. No ownership is taken. */ @@ -31,7 +33,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag; + pops_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 6c58986b534..4895e0d1c92 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -656,7 +656,7 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { if (channel_args->args[i].type == GRPC_ARG_POINTER) { return grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); + (grpc_resource_quota *)channel_args->args[i].value.pointer.p); } else { gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer"); } @@ -666,12 +666,12 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( } static void *rq_copy(void *rq) { - grpc_resource_quota_ref(rq); + grpc_resource_quota_ref((grpc_resource_quota *)rq); return rq; } static void rq_destroy(grpc_exec_ctx *exec_ctx, void *rq) { - grpc_resource_quota_unref_internal(exec_ctx, rq); + grpc_resource_quota_unref_internal(exec_ctx, (grpc_resource_quota *)rq); } static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); } diff --git a/src/core/lib/iomgr/socket_factory_posix.c b/src/core/lib/iomgr/socket_factory_posix.c index 0f82dea570f..c81566575ef 100644 --- a/src/core/lib/iomgr/socket_factory_posix.c +++ b/src/core/lib/iomgr/socket_factory_posix.c @@ -69,11 +69,11 @@ void grpc_socket_factory_unref(grpc_socket_factory *factory) { } static void *socket_factory_arg_copy(void *p) { - return grpc_socket_factory_ref(p); + return grpc_socket_factory_ref((grpc_socket_factory *)p); } static void socket_factory_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_factory_unref(p); + grpc_socket_factory_unref((grpc_socket_factory *)p); } static int socket_factory_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c index 5d6c2c400eb..300ac75b387 100644 --- a/src/core/lib/iomgr/socket_mutator.c +++ b/src/core/lib/iomgr/socket_mutator.c @@ -60,11 +60,11 @@ void grpc_socket_mutator_unref(grpc_socket_mutator *mutator) { } static void *socket_mutator_arg_copy(void *p) { - return grpc_socket_mutator_ref(p); + return grpc_socket_mutator_ref((grpc_socket_mutator *)p); } static void socket_mutator_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_mutator_unref(p); + grpc_socket_mutator_unref((grpc_socket_mutator *)p); } static int socket_mutator_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index ae2c0bf0ae6..04ca44563d6 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -276,7 +276,7 @@ static void timer_thread(void *completed_thread_ptr) { GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); timer_main_loop(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); - timer_thread_cleanup(completed_thread_ptr); + timer_thread_cleanup((completed_thread *)completed_thread_ptr); } static void start_threads(void) { diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 9a02c1d1bba..00b2e68bb5e 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -118,7 +118,7 @@ static grpc_socket_factory *get_socket_factory(const grpc_channel_args *args) { const grpc_arg *arg = grpc_channel_args_find(args, GRPC_ARG_SOCKET_FACTORY); if (arg) { GPR_ASSERT(arg->type == GRPC_ARG_POINTER); - return arg->value.pointer.p; + return (grpc_socket_factory *)arg->value.pointer.p; } } return NULL; diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 321a21a10b7..0764eda0527 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -174,8 +174,8 @@ static const grpc_slice_refcount_vtable new_with_len_vtable = { grpc_slice grpc_slice_new_with_len(void *p, size_t len, void (*destroy)(void *, size_t)) { grpc_slice slice; - new_with_len_slice_refcount *rc = - gpr_malloc(sizeof(new_with_len_slice_refcount)); + new_with_len_slice_refcount *rc = (new_with_len_slice_refcount *)gpr_malloc( + sizeof(new_with_len_slice_refcount)); gpr_ref_init(&rc->refs, 1); rc->rc.vtable = &new_with_len_vtable; rc->rc.sub_refcount = &rc->rc; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 93c512df694..f4e11025b1b 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call; +} parent_call_t; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call; +} child_call_t; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call* */ gpr_atm parent_call_atm; - child_call *child_call; + /* parent_call_t* */ gpr_atm parent_call_atm; + child_call_t *child_call; /* client or server call */ bool is_client; @@ -293,32 +293,32 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl); static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error, bool has_cancelled); -static void add_init_error(grpc_error **composite, grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; +static void add_init_error(grpc_error **composite, grpc_error *new_err) { + if (new_err == GRPC_ERROR_NONE) return; if (*composite == GRPC_ERROR_NONE) *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed"); - *composite = grpc_error_add_child(*composite, new); + *composite = grpc_error_add_child(*composite, new_err); } void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call *get_or_create_parent_call(grpc_call *call) { - parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_or_create_parent_call(grpc_call *call) { + parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call *get_parent_call(grpc_call *call) { - return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_parent_call(grpc_call *call) { + return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -378,15 +378,15 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; if (args->parent_call != NULL) { - child_call *cc = call->child_call = - gpr_arena_alloc(arena, sizeof(child_call)); + child_call_t *cc = call->child_call = + (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); call->child_call->parent = args->parent_call; GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - parent_call *pc = get_or_create_parent_call(args->parent_call); + parent_call_t *pc = get_or_create_parent_call(args->parent_call); gpr_mu_lock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call *pc = get_parent_call(c); + parent_call_t *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status(call, set_status_value_directly, &c->final_info.final_status, - NULL); + get_final_status((grpc_call *)call, set_status_value_directly, + &c->final_info.final_status, NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call *cc = c->child_call; + child_call_t *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call *pc = get_parent_call(cc->parent); + parent_call_t *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -1309,7 +1309,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call *pc = get_parent_call(call); + parent_call_t *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); @@ -1345,7 +1345,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, if (bctl->completion_data.notify_tag.is_closure) { /* unrefs bctl->error */ bctl->call = NULL; - GRPC_CLOSURE_RUN(exec_ctx, bctl->completion_data.notify_tag.tag, error); + GRPC_CLOSURE_RUN( + exec_ctx, (grpc_closure *)bctl->completion_data.notify_tag.tag, error); GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); } else { /* unrefs bctl->error */ @@ -1474,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, bctlp); + process_data_after_md(exec_ctx, (batch_control *)bctlp); } } @@ -1679,11 +1680,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (nops == 0) { if (!is_notify_tag_closure) { GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag)); - grpc_cq_end_op(exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, - free_no_op_completion, NULL, - gpr_malloc(sizeof(grpc_cq_completion))); + grpc_cq_end_op( + exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, + free_no_op_completion, NULL, + (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion))); } else { - GRPC_CLOSURE_SCHED(exec_ctx, notify_tag, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(exec_ctx, (grpc_closure *)notify_tag, GRPC_ERROR_NONE); } error = GRPC_CALL_OK; goto done; From 090aca55bda549a1c80bae8f4770de330b88b4bd Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 12:14:13 -0700 Subject: [PATCH 10/23] Using already type casted variables --- src/core/lib/surface/call.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index f4e11025b1b..de5aa7b6bec 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status((grpc_call *)call, set_status_value_directly, - &c->final_info.final_status, NULL); + get_final_status(c, set_status_value_directly, &c->final_info.final_status, + NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -1475,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, (batch_control *)bctlp); + process_data_after_md(exec_ctx, bctl); } } From 97dab51a3f356a1cb324c9d7e01ed12ba84e5ed4 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 14:27:46 -0700 Subject: [PATCH 11/23] Renaming pops_tag enum --- src/core/lib/iomgr/polling_entity.c | 18 +++++++++--------- src/core/lib/iomgr/polling_entity.h | 8 ++++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c index 74d8794af50..8591a5518e3 100644 --- a/src/core/lib/iomgr/polling_entity.c +++ b/src/core/lib/iomgr/polling_entity.c @@ -25,7 +25,7 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset_set( grpc_pollset_set *pollset_set) { grpc_polling_entity pollent; pollent.pollent.pollset_set = pollset_set; - pollent.tag = POPS_POLLSET_SET; + pollent.tag = GRPC_POLLS_POLLSET_SET; return pollent; } @@ -33,12 +33,12 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset( grpc_pollset *pollset) { grpc_polling_entity pollent; pollent.pollent.pollset = pollset; - pollent.tag = POPS_POLLSET; + pollent.tag = GRPC_POLLS_POLLSET; return pollent; } grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { return pollent->pollent.pollset; } return NULL; @@ -46,23 +46,23 @@ grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { grpc_pollset_set *grpc_polling_entity_pollset_set( grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET_SET) { + if (pollent->tag == GRPC_POLLS_POLLSET_SET) { return pollent->pollent.pollset_set; } return NULL; } bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) { - return pollent->tag == POPS_NONE; + return pollent->tag == GRPC_POLLS_NONE; } void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); @@ -75,10 +75,10 @@ void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 940f10f152d..a161e1fea6f 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,7 +22,11 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" -typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; +typedef enum grpc_pollset_tag { + GRPC_POLLS_NONE, + GRPC_POLLS_POLLSET, + GRPC_POLLS_POLLSET_SET +} grpc_pollset_tag; /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an @@ -33,7 +37,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - pops_tag tag; + grpc_pollset_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( From 72b36736ceaa7a267722cbcfe99a94bb879e3204 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 13 Sep 2017 00:05:29 +0200 Subject: [PATCH 12/23] The input files also need to follow the base argument... --- tools/buildgen/generate_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 559ea1624bc..d29cd02152f 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -85,7 +85,7 @@ for template in reversed(sorted(templates)): test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) - cmd.append(root + '/' + f) + cmd.append(args.base + '/' + root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) jobset.run(pre_jobs, maxjobs=args.jobs) From 49119a4caf48c20ed3655e0e5637129c9ed94481 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 8 Sep 2017 15:29:19 -0700 Subject: [PATCH 13/23] Fix for max_concurrent_streams issue - Call mark_stream_closed before sending trailing metadata --- .../ext/transport/chttp2/transport/chttp2_transport.c | 8 +++++++- .../ext/transport/chttp2/transport/frame_rst_stream.c | 3 +++ src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 + src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3fd701fe2f1..5fb943d3ca8 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,6 +1974,10 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, + GRPC_ERROR_REF(due_to_error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2103,7 +2107,6 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2288,6 +2291,9 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 0133b6efa25..d77097811a9 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,6 +103,9 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 82ff2c8e2cf..e4f2ae84759 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,6 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index fa224a49a45..7512fe122d5 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,6 +386,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, @@ -444,6 +446,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( @@ -519,10 +523,6 @@ void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_REF(error)); s->sending_bytes = 0; } - if (s->sent_trailing_metadata) { - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_REF(error)); - } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end"); } grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &t->outbuf); From 6988dbb45696f385cb3945e3c10b5ba67872ec35 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:34:09 -0700 Subject: [PATCH 14/23] There is no need to remove fail_pending_writes from mark_stream_closed anymore --- .../ext/transport/chttp2/transport/chttp2_transport.c | 8 +------- .../ext/transport/chttp2/transport/frame_rst_stream.c | 3 --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 2 +- src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 5fb943d3ca8..3fd701fe2f1 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,10 +1974,6 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, - GRPC_ERROR_REF(due_to_error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2107,6 +2103,7 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2291,9 +2288,6 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index d77097811a9..0133b6efa25 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,9 +103,6 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index e4f2ae84759..35d52963e22 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); + // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 7512fe122d5..3fa38db4db6 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,13 +386,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); } result.early_results_scheduled |= update_list(exec_ctx, t, s, @@ -446,13 +446,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); now_writing = true; result.early_results_scheduled = true; grpc_chttp2_complete_closure_step( From a6ede3bea1cde14ffd5fc7f9b80a27dcdee345d9 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:37:27 -0700 Subject: [PATCH 15/23] Remove unnecessary comment --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 35d52963e22..82ff2c8e2cf 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,6 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); From 2be06b43e7532ef2e9d1e5709ceb162957e6a926 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 15:24:49 -0700 Subject: [PATCH 16/23] Make unmeasured CPU costs be 1, not 0 Allows capturing 0 as a CPU cost, which we'll be able to leverage to increase concurrency (once data has caught up in a few days) --- tools/run_tests/python_utils/jobset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f3..80e79444ea8 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -224,7 +224,7 @@ class JobResult(object): self.retries = 0 self.message = '' self.cpu_estimated = 1 - self.cpu_measured = 0 + self.cpu_measured = 1 def read_from_start(f): From b2f490c565e87822245c486ce5b04b17adacd52d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 16:25:39 -0700 Subject: [PATCH 17/23] Include limits.h since we're using INT_MAX (affects some compilers) --- src/core/ext/transport/chttp2/transport/flow_control.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index 39aa521029b..0f078e79e90 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -18,6 +18,7 @@ #include "src/core/ext/transport/chttp2/transport/internal.h" +#include #include #include From 707145c78a7e1d65ddf14836adbbb7032172a3a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 10:02:58 +0200 Subject: [PATCH 18/23] mingw already installed on win kokoro workers --- tools/internal_ci/windows/grpc_build_artifacts.bat | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 17d9571d43e..c6e523697ca 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -19,8 +19,6 @@ rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits -pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc - @rem enter repo root cd /d %~dp0\..\..\.. From e97bc51d3966aaa6d8bbcd6ec9238063bba8dad2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 12:26:32 +0200 Subject: [PATCH 19/23] already installed on mac workers --- .../helper_scripts/prepare_build_macos_rc | 13 +------------ tools/internal_ci/macos/grpc_build_artifacts.sh | 12 ------------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index f7fbec93ffa..dd2741e595b 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -31,23 +31,16 @@ ulimit -a pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json -# required to build protobuf -brew install gflags - set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -rvm install ruby-2.4 +rvm use ruby-2.4 rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex -gem install bundler - # cocoapods export LANG=en_US.UTF-8 -gem install cocoapods -gem install xcpretty pod repo update # needed by python # python @@ -56,10 +49,6 @@ pip install virtualenv --user python pip install -U six tox setuptools --user python export PYTHONPATH=/Library/Python/3.4/site-packages -# python 3.4 -wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg -sudo installer -pkg python-3.4.4-macosx10.6.pkg -target / - # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 603c15f210f..09784e3bb47 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -20,25 +20,13 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_macos_rc -# python 3.5 -wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg -sudo installer -pkg python-3.5.2-macosx10.6.pkg -target / - # install cython for all python versions python2.7 -m pip install cython setuptools wheel python3.4 -m pip install cython setuptools wheel python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel -# node-gyp (needed for node artifacts) -npm install -g node-gyp - -# php dependencies: pecl -curl -O http://pear.php.net/go-pear.phar -sudo php -d detect_unicode=0 go-pear.phar - # needed to build ruby artifacts -gem install rake-compiler wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh bash build_ruby_environment_macos.sh From e6d2e8f16bd2ebbfd112b09e589205191211a2c8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 06:10:18 -0700 Subject: [PATCH 20/23] Fix case on internal type names Should allow microbenchmarks to get running again --- tools/profiling/microbenchmarks/bm2bq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 6c5040de56d..9f9b672f759 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -30,7 +30,7 @@ columns = [] for row in json.loads( subprocess.check_output([ 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: - columns.append((row['name'], row['type'])) + columns.append((row['name'], row['type'].lower())) SANITIZE = { 'integer': int, From 70bbb6b1176626346efc7c7549fb027896f8d8c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:04:30 -0700 Subject: [PATCH 21/23] Fix schema generation --- src/core/lib/debug/stats_data_bq_schema.sql | 70 ++++++++++----------- tools/codegen/core/gen_stats_data.py | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 7291bbf0709..53547692918 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,35 +1,35 @@ -client_calls_created_per_iteration:INTEGER, -server_calls_created_per_iteration:INTEGER, -syscall_poll_per_iteration:INTEGER, -syscall_wait_per_iteration:INTEGER, -histogram_slow_lookups_per_iteration:INTEGER, -syscall_write_per_iteration:INTEGER, -syscall_read_per_iteration:INTEGER, -tcp_backup_pollers_created_per_iteration:INTEGER, -tcp_backup_poller_polls_per_iteration:INTEGER, -http2_op_batches_per_iteration:INTEGER, -http2_op_cancel_per_iteration:INTEGER, -http2_op_send_initial_metadata_per_iteration:INTEGER, -http2_op_send_message_per_iteration:INTEGER, -http2_op_send_trailing_metadata_per_iteration:INTEGER, -http2_op_recv_initial_metadata_per_iteration:INTEGER, -http2_op_recv_message_per_iteration:INTEGER, -http2_op_recv_trailing_metadata_per_iteration:INTEGER, -http2_settings_writes_per_iteration:INTEGER, -http2_pings_sent_per_iteration:INTEGER, -http2_writes_begun_per_iteration:INTEGER, -http2_writes_offloaded_per_iteration:INTEGER, -http2_writes_continued_per_iteration:INTEGER, -http2_partial_writes_per_iteration:INTEGER, -combiner_locks_initiated_per_iteration:INTEGER, -combiner_locks_scheduled_items_per_iteration:INTEGER, -combiner_locks_scheduled_final_items_per_iteration:INTEGER, -combiner_locks_offloaded_per_iteration:INTEGER, -executor_scheduled_short_items_per_iteration:INTEGER, -executor_scheduled_long_items_per_iteration:INTEGER, -executor_scheduled_to_self_per_iteration:INTEGER, -executor_wakeup_initiated_per_iteration:INTEGER, -executor_queue_drained_per_iteration:INTEGER, -executor_push_retries_per_iteration:INTEGER, -server_requested_calls_per_iteration:INTEGER, -server_slowpath_requests_queued_per_iteration:INTEGER +client_calls_created_per_iteration:FLOAT, +server_calls_created_per_iteration:FLOAT, +syscall_poll_per_iteration:FLOAT, +syscall_wait_per_iteration:FLOAT, +histogram_slow_lookups_per_iteration:FLOAT, +syscall_write_per_iteration:FLOAT, +syscall_read_per_iteration:FLOAT, +tcp_backup_pollers_created_per_iteration:FLOAT, +tcp_backup_poller_polls_per_iteration:FLOAT, +http2_op_batches_per_iteration:FLOAT, +http2_op_cancel_per_iteration:FLOAT, +http2_op_send_initial_metadata_per_iteration:FLOAT, +http2_op_send_message_per_iteration:FLOAT, +http2_op_send_trailing_metadata_per_iteration:FLOAT, +http2_op_recv_initial_metadata_per_iteration:FLOAT, +http2_op_recv_message_per_iteration:FLOAT, +http2_op_recv_trailing_metadata_per_iteration:FLOAT, +http2_settings_writes_per_iteration:FLOAT, +http2_pings_sent_per_iteration:FLOAT, +http2_writes_begun_per_iteration:FLOAT, +http2_writes_offloaded_per_iteration:FLOAT, +http2_writes_continued_per_iteration:FLOAT, +http2_partial_writes_per_iteration:FLOAT, +combiner_locks_initiated_per_iteration:FLOAT, +combiner_locks_scheduled_items_per_iteration:FLOAT, +combiner_locks_scheduled_final_items_per_iteration:FLOAT, +combiner_locks_offloaded_per_iteration:FLOAT, +executor_scheduled_short_items_per_iteration:FLOAT, +executor_scheduled_long_items_per_iteration:FLOAT, +executor_scheduled_to_self_per_iteration:FLOAT, +executor_wakeup_initiated_per_iteration:FLOAT, +executor_queue_drained_per_iteration:FLOAT, +executor_push_retries_per_iteration:FLOAT, +server_requested_calls_per_iteration:FLOAT, +server_slowpath_requests_queued_per_iteration:FLOAT diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 877a5d9d430..8359734c848 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -405,6 +405,6 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: columns = [] for counter in inst_map['Counter']: - columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) From f2cbd6c81eab267fb8c04121ff76ffbbe689dfa1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:31:16 -0700 Subject: [PATCH 22/23] Fix qps_test histogram python code --- tools/run_tests/performance/massage_qps_stats_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index 400a0c82bfd..a2fe4ae6c34 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -23,7 +23,7 @@ def _threshold_for_count_below(buckets, boundaries, count_below): if count_so_far == count_below: # this bucket hits the threshold exactly... we should be midway through # any run of zero values following the bucket - for upper_idx in range(lower_idx + 1, num_buckets): + for upper_idx in range(lower_idx + 1, len(buckets)): if buckets[upper_idx] != 0: break return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 From 5160155005c7d0f1ee9b855dfbe6aa45d8602bf1 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 09:31:07 -0700 Subject: [PATCH 23/23] Upload Kokoro MacOS master results --- tools/internal_ci/macos/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/grpc_basictests_opt.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index f058f0c7e47..53bda1ff0a6 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index 5048baaf483..d359eb601a0 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" }