From 37a907e471f9a2e893cb1b95dc2808bb4c57049b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 13 May 2016 13:49:43 -0700 Subject: [PATCH 1/3] add support for CLIENT_LANGUAGE setting --- tools/run_tests/performance/scenario_config.py | 10 ++++++++++ tools/run_tests/run_performance_tests.py | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 4fe66dff41d..44fce6106ee 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -81,6 +81,7 @@ def remove_nonproto_fields(scenario): """Remove special-purpose that contains some extra info about the scenario but don't belong to the ScenarioConfig protobuf message""" scenario.pop('CATEGORIES', None) + scenario.pop('CLIENT_LANGUAGE', None) scenario.pop('SERVER_LANGUAGE', None) return scenario @@ -90,6 +91,7 @@ def _ping_pong_scenario(name, rpc_type, secure=True, use_generic_payload=False, use_unconstrained_client=False, + client_language=None, server_language=None, server_core_limit=0, async_server_threads=0, @@ -142,6 +144,9 @@ def _ping_pong_scenario(name, rpc_type, scenario['client_config']['client_channels'] = 1 scenario['client_config']['async_client_threads'] = 1 + if client_language: + # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py + scenario['CLIENT_LANGUAGE'] = client_language if server_language: # the SERVER_LANGUAGE field is recognized by run_performance_tests.py scenario['SERVER_LANGUAGE'] = server_language @@ -277,6 +282,11 @@ class CSharpLanguage: client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1) + yield _ping_pong_scenario( + 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + use_unconstrained_client=True, client_language='c++') + def __str__(self): return 'csharp' diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index a7728e7f7d5..181d62bf4a3 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -304,7 +304,11 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) + custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields(scenario_json) + if custom_server_lang and custom_client_lang: + raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE' + 'in the same scenario') if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print 'Warning: Skipping scenario %s as' % scenario_json['name'] @@ -314,6 +318,16 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', for idx in range(0, scenario_json['num_servers']): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][idx] + if custom_client_lang: + if not workers_by_lang.get(custom_client_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('CLIENT_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_client_lang) + continue + for idx in range(scenario_json['num_servers'], len(workers)): + # replace all client workers by workers of a different language, + # leave num_server workers as they are server workers. + workers[idx] = workers_by_lang[custom_client_lang][idx] scenario = create_scenario_jobspec(scenario_json, workers, remote_host=remote_host, @@ -360,8 +374,8 @@ argp.add_argument('--bq_result_table', default=None, type=str, help='Bigquery "dataset.table" to upload results to.') argp.add_argument('--category', choices=['smoketest','all'], - default='smoketest', - help='Select a category of tests to run. Smoketest runs by default.') + default='all', + help='Select a category of tests to run.') argp.add_argument('--netperf', default=False, action='store_const', From 5cbccd078144455bad20e78dfc081f131b088a56 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 13 May 2016 16:26:42 -0700 Subject: [PATCH 2/3] set client params differently for sync and async client --- .../run_tests/performance/scenario_config.py | 101 +++++++++++------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 44fce6106ee..77b158f27e1 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -69,6 +69,10 @@ DEEP=100 # wide is the number of client channels in multi-channel tests (1 otherwise) WIDE=64 +# For most synchronous clients, DEEP*WIDE threads will be created. +SYNC_DEEP=10 +SYNC_WIDE=8 + def _get_secargs(is_secure): if is_secure: @@ -90,7 +94,7 @@ def _ping_pong_scenario(name, rpc_type, client_type, server_type, secure=True, use_generic_payload=False, - use_unconstrained_client=False, + unconstrained_client=None, client_language=None, server_language=None, server_core_limit=0, @@ -132,12 +136,19 @@ def _ping_pong_scenario(name, rpc_type, # For proto payload, only the client should get the config. scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD - if use_unconstrained_client: + if unconstrained_client: + if unconstrained_client == 'async': + deep = DEEP + wide = WIDE + elif unconstrained_client == 'sync': + deep = SYNC_DEEP + wide = SYNC_WIDE + else: + raise Exception('Illegal value of unconstrained_client option.') + scenario['num_clients'] = 0 # use as many client as available. - # TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads - # and that's probably too much (at least for wrapped languages). - scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP - scenario['client_config']['client_channels'] = WIDE + scenario['client_config']['outstanding_rpcs_per_channel'] = deep + scenario['client_config']['client_channels'] = wide scenario['client_config']['async_client_threads'] = 0 else: scenario['client_config']['outstanding_rpcs_per_channel'] = 1 @@ -201,27 +212,27 @@ class CXXLanguage: yield _ping_pong_scenario( 'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure, categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_unconstrained_client=True, use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, secure=secure, categories=smoketest_categories) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_unconstrained_client=True, use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, server_core_limit=1, async_server_threads=1, secure=secure) @@ -263,13 +274,13 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', categories=[SMOKETEST]) yield _ping_pong_scenario( 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True) + unconstrained_client='async') yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', @@ -282,10 +293,21 @@ class CSharpLanguage: client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1) + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='async', server_language='c++') + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='sync', server_language='c++') + yield _ping_pong_scenario( 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, client_language='c++') + unconstrained_client='async', client_language='c++') + def __str__(self): return 'csharp' @@ -323,14 +345,14 @@ class NodeLanguage: yield _ping_pong_scenario( 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', categories=[SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( # 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - # use_unconstrained_client=True) + # unconstrained_client='async') # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -379,19 +401,15 @@ class PythonLanguage: client_type='SYNC_CLIENT', server_type='SYNC_SERVER', categories=[SMOKETEST]) - # TODO(jtattermusch): - # The qps_worker server gets thread starved with ~6400 threads, the GIL - # enforces that a single thread runs at a time, with no way to set thread - # priority. Re-evaluate after changing DEEP and WIDE. - #yield _ping_pong_scenario( - # 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', - # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - # use_unconstrained_client=True) + yield _ping_pong_scenario( + 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + unconstrained_client='sync') yield _ping_pong_scenario( 'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='SYNC_SERVER', - use_unconstrained_client=True) + unconstrained_client='async') yield _ping_pong_scenario( 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', @@ -430,17 +448,15 @@ class RubyLanguage: client_type='SYNC_CLIENT', server_type='SYNC_SERVER', categories=[SMOKETEST]) - # TODO: scenario reports QPS of 0.0 - #yield _ping_pong_scenario( - # 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', - # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - # use_unconstrained_client=True) + yield _ping_pong_scenario( + 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + unconstrained_client='sync') - # TODO: scenario reports QPS of 0.0 - #yield _ping_pong_scenario( - # 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', - # client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - # use_unconstrained_client=True) + yield _ping_pong_scenario( + 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + unconstrained_client='sync') yield _ping_pong_scenario( 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', @@ -502,26 +518,26 @@ class JavaLanguage: yield _ping_pong_scenario( 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, categories=smoketest_categories) yield _ping_pong_scenario( 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) yield _ping_pong_scenario( 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_unconstrained_client=True, use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) yield _ping_pong_scenario( 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_unconstrained_client=True, use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, async_server_threads=1, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) @@ -570,25 +586,28 @@ class GoLanguage: secure=secure, categories=smoketest_categories) + # unconstrained_client='async' is intended (client uses goroutines) yield _ping_pong_scenario( 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure, categories=smoketest_categories) + # unconstrained_client='async' is intended (client uses goroutines) yield _ping_pong_scenario( 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - use_unconstrained_client=True, + unconstrained_client='async', secure=secure) + # unconstrained_client='async' is intended (client uses goroutines) # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. yield _ping_pong_scenario( 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_unconstrained_client=True, use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, secure=secure) # TODO(jtattermusch): add scenarios go vs C++ From 9be075c9116e63720dfbe297bc18924e2c953c32 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 17 May 2016 10:46:59 -0700 Subject: [PATCH 3/3] Fix leak in port deallocation --- test/core/util/port_posix.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index f13960156fa..265e0acee14 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -89,6 +89,7 @@ static int free_chosen_port(int port) { grpc_free_port_using_server(env, port); } } + gpr_free(env); return found; }