Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #6462 from jtattermusch/benchmarking_unify_scenarios

Browse Source 
Improve benchmarking scenarios and enable "smoketest only" suite.
pull/6524/head
Jan Tattermusch 9 years ago
parent dcd8d7cd99 abd1b38885
commit 92a9156970
4 changed files with 465 additions and 553 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 10
      test/cpp/qps/gen_build_yaml.py
  2. 858
      tools/run_tests/performance/scenario_config.py
  3. 50
      tools/run_tests/run_performance_tests.py
  4. 100
      tools/run_tests/tests.json

10
test/cpp/qps/gen_build_yaml.py
Unescape View File

@ -43,12 +43,16 @@ sys.path.append(run_tests_root)
import performance.scenario_config as scenario_config
def _scenario_json_string(scenario_json):
return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
print yaml.dump({
'tests': [
{
'name': 'json_run_localhost',
'shortname': 'json_run_localhost:%s' % js['name'],
'args': ['--scenario_json', pipes.quote(json.dumps(js))],
'shortname': 'json_run_localhost:%s' % scenario_json['name'],
'args': ['--scenario_json',
pipes.quote(_scenario_json_string(scenario_json))],
'ci_platforms': ['linux', 'mac', 'posix', 'windows'],
'platforms': ['linux', 'mac', 'posix', 'windows'],
'flaky': False,
@ -58,6 +62,6 @@ print yaml.dump({
'cpu_cost': 1000.0,
'exclude_configs': []
}
for js in scenario_config.CXXLanguage().scenarios()
for scenario_json in scenario_config.CXXLanguage().scenarios()
]
})

858
tools/run_tests/performance/scenario_config.py
Unescape View File

@ -34,6 +34,8 @@ WARMUP_SECONDS=5
JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in.
BENCHMARK_SECONDS=30
SMOKETEST='smoketest'
SECURE_SECARGS = {'use_test_ca': True,
'server_host_override': 'foo.test.google.fr'}
@ -69,6 +71,86 @@ DEEP=100
WIDE=64
def _get_secargs(is_secure):
if is_secure:
return SECURE_SECARGS
else:
return None
def remove_nonproto_fields(scenario):
"""Remove special-purpose that contains some extra info about the scenario
but don't belong to the ScenarioConfig protobuf message"""
scenario.pop('CATEGORIES', None)
scenario.pop('SERVER_LANGUAGE', None)
return scenario
def _ping_pong_scenario(name, rpc_type,
client_type, server_type,
secure=True,
use_generic_payload=False,
use_unconstrained_client=False,
server_language=None,
server_core_limit=0,
async_server_threads=0,
warmup_seconds=WARMUP_SECONDS,
categories=[]):
"""Creates a basic ping pong scenario."""
scenario = {
'name': name,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': client_type,
'security_params': _get_secargs(secure),
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': rpc_type,
'load_params': {
'closed_loop': {}
},
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': server_type,
'security_params': _get_secargs(secure),
'core_limit': server_core_limit,
'async_server_threads': async_server_threads,
},
'warmup_seconds': warmup_seconds,
'benchmark_seconds': BENCHMARK_SECONDS
}
if use_generic_payload:
if server_type != 'ASYNC_GENERIC_SERVER':
raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
scenario['client_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD
scenario['server_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD
else:
# For proto payload, only the client should get the config.
scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD
if use_unconstrained_client:
scenario['num_clients'] = 0 # use as many client as available.
# TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads
# and that's probably too much (at least for wrapped languages).
scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP
scenario['client_config']['client_channels'] = WIDE
scenario['client_config']['async_client_threads'] = 0
else:
scenario['client_config']['outstanding_rpcs_per_channel'] = 1
scenario['client_config']['client_channels'] = 1
scenario['client_config']['async_client_threads'] = 1
if server_language:
# the SERVER_LANGUAGE field is recognized by run_performance_tests.py
scenario['SERVER_LANGUAGE'] = server_language
if categories:
scenario['CATEGORIES'] = categories
return scenario
class CXXLanguage:
def __init__(self):
@ -83,205 +165,64 @@ class CXXLanguage:
def scenarios(self):
# TODO(ctiller): add 70% load latency test
for secure in [True, False]:
if secure:
secstr = 'secure'
secargs = SECURE_SECARGS
else:
secstr = 'insecure'
secargs = None
yield {
'name': 'cpp_generic_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
'async_server_threads': 0,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_protobuf_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_protobuf_sync_unary_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 0,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'cpp_protobuf_async_unary_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
secstr = 'secure' if secure else 'insecure'
smoketest_categories = [SMOKETEST] if secure else None
yield _ping_pong_scenario(
'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True, server_core_limit=1, async_server_threads=1,
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_core_limit=1, async_server_threads=1,
secure=secure)
yield _ping_pong_scenario(
'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_core_limit=1, async_server_threads=1,
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_core_limit=1, async_server_threads=1,
secure=secure)
yield _ping_pong_scenario(
'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_core_limit=SINGLE_MACHINE_CORES/2,
use_unconstrained_client=True,
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_core_limit=SINGLE_MACHINE_CORES/2,
use_unconstrained_client=True,
secure=secure)
yield _ping_pong_scenario(
'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
server_core_limit=SINGLE_MACHINE_CORES/2,
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
server_core_limit=1, async_server_threads=1,
secure=secure)
def __str__(self):
return 'c++'
@ -299,113 +240,48 @@ class CSharpLanguage:
return 100
def scenarios(self):
secargs = SECURE_SECARGS
yield {
'name': 'csharp_generic_async_streaming_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 0,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'csharp_protobuf_async_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': secargs,
'core_limit': 1,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS,
'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py
}
yield _ping_pong_scenario(
'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True,
categories=[SMOKETEST])
yield _ping_pong_scenario(
'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
yield _ping_pong_scenario(
'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
categories=[SMOKETEST])
yield _ping_pong_scenario(
'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='ASYNC_SERVER')
# TODO(jtattermusch): scenario works locally but fails on jenkins
#yield _ping_pong_scenario(
# 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# use_unconstrained_client=True,
# categories=[SMOKETEST])
# TODO(jtattermusch): scenario works locally but fails on jenkins
#yield _ping_pong_scenario(
# 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1,
categories=[SMOKETEST])
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1)
def __str__(self):
return 'csharp'
@ -424,34 +300,45 @@ class NodeLanguage:
return 200
def scenarios(self):
# TODO(jtattermusch): add more scenarios
secargs = SECURE_SECARGS
yield {
'name': 'node_protobuf_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_generic_async_streaming_ping_pong', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
# use_generic_payload=True)
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
yield _ping_pong_scenario(
'node_protobuf_unary_ping_pong', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
categories=[SMOKETEST])
yield _ping_pong_scenario(
'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
categories=[SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# use_unconstrained_client=True)
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# server_language='c++', server_core_limit=1, async_server_threads=1)
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# server_language='c++', server_core_limit=1, async_server_threads=1)
def __str__(self):
return 'node'
@ -468,114 +355,52 @@ class PythonLanguage:
return 500
def scenarios(self):
yield {
'name': 'python_to_cpp_protobuf_streaming_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': SECURE_SECARGS,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': SECURE_SECARGS,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS,
'SERVER_LANGUAGE': 'c++'
}
yield {
'name': 'python_protobuf_sync_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': SECURE_SECARGS,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': SECURE_SECARGS,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS,
}
yield {
'name': 'python_protobuf_async_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': SECURE_SECARGS,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': SECURE_SECARGS,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS,
}
yield {
'name': 'python_to_cpp_single_channel_throughput',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': SECURE_SECARGS,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': BIG_GENERIC_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': SECURE_SECARGS,
'core_limit': SINGLE_MACHINE_CORES/2,
'async_server_threads': 1,
'payload_config': BIG_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS,
'SERVER_LANGUAGE': 'c++'
}
# TODO(jtattermusch): this scenario reports QPS 0.0
yield _ping_pong_scenario(
'python_generic_async_streaming_ping_pong', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True,
categories=[SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'python_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
yield _ping_pong_scenario(
'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1,
categories=[SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# server_language='c++', server_core_limit=1, async_server_threads=1)
def __str__(self):
return 'python'
@ -592,34 +417,37 @@ class RubyLanguage:
return 300
def scenarios(self):
# TODO(jtattermusch): add more scenarios
secargs = SECURE_SECARGS
yield {
'name': 'ruby_protobuf_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield _ping_pong_scenario(
'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
yield _ping_pong_scenario(
'ruby_protobuf_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
# TODO: scenario reports QPS of 0.0
#yield _ping_pong_scenario(
# 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
# TODO: scenario reports QPS of 0.0
#yield _ping_pong_scenario(
# 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1)
yield _ping_pong_scenario(
'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1)
def __str__(self):
return 'ruby'
@ -638,41 +466,63 @@ class JavaLanguage:
return 400
def scenarios(self):
# TODO(jtattermusch): add more scenarios
for secure in [True, False]:
if secure:
secstr = 'secure'
secargs = SECURE_SECARGS
else:
secstr = 'insecure'
secargs = None
yield {
'name': 'java_protobuf_unary_ping_pong_%s' % secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'SYNC_CLIENT',
'security_params': secargs,
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
'async_server_threads': 1,
},
'warmup_seconds': JAVA_WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
secstr = 'secure' if secure else 'insecure'
smoketest_categories = [SMOKETEST] if secure else None
yield _ping_pong_scenario(
'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True, async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
categories=smoketest_categories)
yield _ping_pong_scenario(
'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
categories=smoketest_categories)
yield _ping_pong_scenario(
'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
categories=smoketest_categories)
yield _ping_pong_scenario(
'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
# TODO(jtattermusch): add scenarios java vs C++
def __str__(self):
return 'java'

50
tools/run_tests/run_performance_tests.py
Unescape View File

@ -89,7 +89,7 @@ def create_qpsworker_job(language, shortname=None,
jobspec = jobset.JobSpec(
cmdline=cmdline,
shortname=shortname,
timeout_seconds=30*60)
timeout_seconds=2*60*60)
return QpsWorkerJob(jobspec, language, host_and_port)
@ -244,7 +244,7 @@ def start_qpsworkers(languages, worker_hosts):
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
bq_result_table=None):
category='all', bq_result_table=None):
"""Create jobspecs for scenarios to run."""
all_workers = [worker
for workers in workers_by_lang.values()
@ -253,25 +253,26 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json['name']):
workers = workers_by_lang[str(language)]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language. It doesn't belong to the Scenario
# schema, so we also need to remove it.
custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None)
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json['name']
print('SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
scenario = create_scenario_jobspec(scenario_json,
workers,
remote_host=remote_host,
bq_result_table=bq_result_table)
scenarios.append(scenario)
if category in scenario_json.get('CATEGORIES', []) or category == 'all':
workers = workers_by_lang[str(language)]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json['name']
print('SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
scenario = create_scenario_jobspec(scenario_json,
workers,
remote_host=remote_host,
bq_result_table=bq_result_table)
scenarios.append(scenario)
# the very last scenario requests shutting down the workers.
scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
@ -298,7 +299,7 @@ argp = argparse.ArgumentParser(description='Run performance tests.')
argp.add_argument('-l', '--language',
choices=['all'] + sorted(scenario_config.LANGUAGES.keys()),
nargs='+',
default=['all'],
required=True,
help='Languages to benchmark.')
argp.add_argument('--remote_driver_host',
default=None,
@ -311,6 +312,10 @@ argp.add_argument('-r', '--regex', default='.*', type=str,
help='Regex to select scenarios to run.')
argp.add_argument('--bq_result_table', default=None, type=str,
help='Bigquery "dataset.table" to upload results to.')
argp.add_argument('--category',
choices=['smoketest','all'],
default='smoketest',
help='Select a category of tests to run. Smoketest runs by default.')
args = argp.parse_args()
@ -354,6 +359,7 @@ try:
workers_by_lang=worker_addresses,
remote_host=args.remote_driver_host,
regex=args.regex,
category=args.category,
bq_result_table=args.bq_result_table)
if not scenarios:
raise Exception('No scenarios to run')

100
tools/run_tests/tests.json
Unescape View File

@ -22942,7 +22942,7 @@
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -22963,12 +22963,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure"
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -22989,12 +22989,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
"shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -23015,12 +23015,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure"
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23041,12 +23041,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure"
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23067,12 +23067,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure"
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23093,7 +23093,33 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure"
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1000.0,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
},
{
"args": [
@ -23124,7 +23150,7 @@
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -23145,12 +23171,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure"
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -23171,12 +23197,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
"shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
"'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
],
"boringssl": true,
"ci_platforms": [
@ -23197,12 +23223,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure"
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23223,12 +23249,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure"
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23249,12 +23275,12 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure"
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
"'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
@ -23275,7 +23301,33 @@
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure"
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure"
},
{
"args": [
"--scenario_json",
"'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
],
"boringssl": true,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1000.0,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
},
{
"args": [

Write Preview
Loading…
Cancel
Save