Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #6590 from jtattermusch/perf_script_improvements

Browse Source 
Perf script improvements
pull/6626/head
Jan Tattermusch 9 years ago
parent 8010daaf2b 5cbccd0781
commit d5ed5a98db
2 changed files with 85 additions and 42 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 109
      tools/run_tests/performance/scenario_config.py
  2. 18
      tools/run_tests/run_performance_tests.py

109
tools/run_tests/performance/scenario_config.py
Unescape View File

@ -69,6 +69,10 @@ DEEP=100
# wide is the number of client channels in multi-channel tests (1 otherwise)
WIDE=64
# For most synchronous clients, DEEP*WIDE threads will be created.
SYNC_DEEP=10
SYNC_WIDE=8
def _get_secargs(is_secure):
if is_secure:
@ -81,6 +85,7 @@ def remove_nonproto_fields(scenario):
"""Remove special-purpose that contains some extra info about the scenario
but don't belong to the ScenarioConfig protobuf message"""
scenario.pop('CATEGORIES', None)
scenario.pop('CLIENT_LANGUAGE', None)
scenario.pop('SERVER_LANGUAGE', None)
return scenario
@ -89,7 +94,8 @@ def _ping_pong_scenario(name, rpc_type,
client_type, server_type,
secure=True,
use_generic_payload=False,
use_unconstrained_client=False,
unconstrained_client=None,
client_language=None,
server_language=None,
server_core_limit=0,
async_server_threads=0,
@ -130,18 +136,28 @@ def _ping_pong_scenario(name, rpc_type,
# For proto payload, only the client should get the config.
scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD
if use_unconstrained_client:
if unconstrained_client:
if unconstrained_client == 'async':
deep = DEEP
wide = WIDE
elif unconstrained_client == 'sync':
deep = SYNC_DEEP
wide = SYNC_WIDE
else:
raise Exception('Illegal value of unconstrained_client option.')
scenario['num_clients'] = 0 # use as many client as available.
# TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads
# and that's probably too much (at least for wrapped languages).
scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP
scenario['client_config']['client_channels'] = WIDE
scenario['client_config']['outstanding_rpcs_per_channel'] = deep
scenario['client_config']['client_channels'] = wide
scenario['client_config']['async_client_threads'] = 0
else:
scenario['client_config']['outstanding_rpcs_per_channel'] = 1
scenario['client_config']['client_channels'] = 1
scenario['client_config']['async_client_threads'] = 1
if client_language:
# the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
scenario['CLIENT_LANGUAGE'] = client_language
if server_language:
# the SERVER_LANGUAGE field is recognized by run_performance_tests.py
scenario['SERVER_LANGUAGE'] = server_language
@ -196,27 +212,27 @@ class CXXLanguage:
yield _ping_pong_scenario(
'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure)
yield _ping_pong_scenario(
'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
unconstrained_client='async', use_generic_payload=True,
secure=secure,
categories=smoketest_categories)
yield _ping_pong_scenario(
'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
unconstrained_client='async', use_generic_payload=True,
server_core_limit=1, async_server_threads=1,
secure=secure)
@ -258,13 +274,13 @@ class CSharpLanguage:
yield _ping_pong_scenario(
'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
categories=[SMOKETEST])
yield _ping_pong_scenario(
'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True)
unconstrained_client='async')
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@ -277,6 +293,22 @@ class CSharpLanguage:
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1)
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
unconstrained_client='async', server_language='c++')
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
unconstrained_client='sync', server_language='c++')
yield _ping_pong_scenario(
'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
unconstrained_client='async', client_language='c++')
def __str__(self):
return 'csharp'
@ -313,14 +345,14 @@ class NodeLanguage:
yield _ping_pong_scenario(
'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
categories=[SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
# 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
# use_unconstrained_client=True)
# unconstrained_client='async')
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
@ -369,19 +401,15 @@ class PythonLanguage:
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
# TODO(jtattermusch):
# The qps_worker server gets thread starved with ~6400 threads, the GIL
# enforces that a single thread runs at a time, with no way to set thread
# priority. Re-evaluate after changing DEEP and WIDE.
#yield _ping_pong_scenario(
# 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
unconstrained_client='sync')
yield _ping_pong_scenario(
'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='SYNC_SERVER',
use_unconstrained_client=True)
unconstrained_client='async')
yield _ping_pong_scenario(
'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@ -420,17 +448,15 @@ class RubyLanguage:
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
# TODO: scenario reports QPS of 0.0
#yield _ping_pong_scenario(
# 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
unconstrained_client='sync')
# TODO: scenario reports QPS of 0.0
#yield _ping_pong_scenario(
# 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
# client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
# use_unconstrained_client=True)
yield _ping_pong_scenario(
'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
unconstrained_client='sync')
yield _ping_pong_scenario(
'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@ -492,26 +518,26 @@ class JavaLanguage:
yield _ping_pong_scenario(
'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
categories=smoketest_categories)
yield _ping_pong_scenario(
'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
unconstrained_client='async', use_generic_payload=True,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
yield _ping_pong_scenario(
'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
unconstrained_client='async', use_generic_payload=True,
async_server_threads=1,
secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
@ -560,25 +586,28 @@ class GoLanguage:
secure=secure,
categories=smoketest_categories)
# unconstrained_client='async' is intended (client uses goroutines)
yield _ping_pong_scenario(
'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure,
categories=smoketest_categories)
# unconstrained_client='async' is intended (client uses goroutines)
yield _ping_pong_scenario(
'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
use_unconstrained_client=True,
unconstrained_client='async',
secure=secure)
# unconstrained_client='async' is intended (client uses goroutines)
# ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
# but that's mostly because of lack of better name of the enum value.
yield _ping_pong_scenario(
'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_unconstrained_client=True, use_generic_payload=True,
unconstrained_client='async', use_generic_payload=True,
secure=secure)
# TODO(jtattermusch): add scenarios go vs C++

18
tools/run_tests/run_performance_tests.py
Unescape View File

@ -304,7 +304,11 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
if custom_server_lang and custom_client_lang:
raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE'
'in the same scenario')
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json['name']
@ -314,6 +318,16 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
if custom_client_lang:
if not workers_by_lang.get(custom_client_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json['name']
print('CLIENT_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_client_lang)
continue
for idx in range(scenario_json['num_servers'], len(workers)):
# replace all client workers by workers of a different language,
# leave num_server workers as they are server workers.
workers[idx] = workers_by_lang[custom_client_lang][idx]
scenario = create_scenario_jobspec(scenario_json,
workers,
remote_host=remote_host,
@ -360,8 +374,8 @@ argp.add_argument('--bq_result_table', default=None, type=str,
help='Bigquery "dataset.table" to upload results to.')
argp.add_argument('--category',
choices=['smoketest','all'],
default='smoketest',
help='Select a category of tests to run. Smoketest runs by default.')
default='all',
help='Select a category of tests to run.')
argp.add_argument('--netperf',
default=False,
action='store_const',

Write Preview
Loading…
Cancel
Save