From 86cf6dcb1cda3aefa3e789e68ce825b9e3e381f4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 13 Sep 2016 10:33:29 -0700 Subject: [PATCH] Remove some scenarios we can't run yet --- .../run_tests/performance/scenario_config.py | 2 +- tools/run_tests/tests.json | 324 ------------------ 2 files changed, 1 insertion(+), 325 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 5074b939114..a43a6b6b36d 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -257,7 +257,7 @@ class CXXLanguage: for channels in geometric_progression(1, 500, math.sqrt(10)): for outstanding in geometric_progression(1, 20000, math.sqrt(10)): - if synchronicity == 'sync' and outstanding > 1000: continue + if synchronicity == 'sync' and outstanding > 400: continue if outstanding < channels: continue yield _ping_pong_scenario( 'cpp_protobuf_%s_unary_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, secstr, channels, outstanding), diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index c7b712e86dc..a7fb79ec9b9 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -29919,33 +29919,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_1_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_1_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1000, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_1_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -30081,33 +30054,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_3_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_3_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 3, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 334, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_3_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -30216,33 +30162,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_10_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_10_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 10, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_10_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -30324,33 +30243,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_32_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_32_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 32, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 32, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_32_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -30405,33 +30297,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_100_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_100_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 100, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_100_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -30459,33 +30324,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_316_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_316_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 316, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 4, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_316_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -31998,33 +31836,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_1_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_1_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1000, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_1_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -32160,33 +31971,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_3_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_3_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 3, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 334, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_3_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -32295,33 +32079,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_10_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_10_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 10, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_10_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -32403,33 +32160,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_32_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_32_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 32, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 32, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_32_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -32484,33 +32214,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_100_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_100_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 100, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 10, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_100_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json", @@ -32538,33 +32241,6 @@ "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_316_channels_316_outstanding", "timeout_seconds": 180 }, - { - "args": [ - "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_316_channels_1000_outstanding\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 316, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 4, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" - ], - "boringssl": true, - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 8, - "defaults": "boringssl", - "exclude_configs": [], - "flaky": false, - "language": "c++", - "name": "json_run_localhost", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_316_channels_1000_outstanding", - "timeout_seconds": 180 - }, { "args": [ "--scenarios_json",