Merge pull request #6590 from jtattermusch/perf_script_improvements

Perf script improvements

tools/run_tests/performance/scenario_config.py

@@ -69,6 +69,10 @@ DEEP=100
 # wide is the number of client channels in multi-channel tests (1 otherwise)
 WIDE=64
 
+# For most synchronous clients, DEEP*WIDE threads will be created.
+SYNC_DEEP=10
+SYNC_WIDE=8
+
 
 def _get_secargs(is_secure):
   if is_secure:
@@ -81,6 +85,7 @@ def remove_nonproto_fields(scenario):
   """Remove special-purpose that contains some extra info about the scenario
   but don't belong to the ScenarioConfig protobuf message"""
   scenario.pop('CATEGORIES', None)
+  scenario.pop('CLIENT_LANGUAGE', None)
   scenario.pop('SERVER_LANGUAGE', None)
   return scenario
 
@@ -89,7 +94,8 @@ def _ping_pong_scenario(name, rpc_type,
                         client_type, server_type,
                         secure=True,
                         use_generic_payload=False,
-                        use_unconstrained_client=False,
+                        unconstrained_client=None,
+                        client_language=None,
                         server_language=None,
                         server_core_limit=0,
                         async_server_threads=0,
@@ -130,18 +136,28 @@ def _ping_pong_scenario(name, rpc_type,
     # For proto payload, only the client should get the config.
     scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD
 
-  if use_unconstrained_client:
+  if unconstrained_client:
+    if unconstrained_client == 'async':
+      deep = DEEP
+      wide = WIDE
+    elif unconstrained_client == 'sync':
+      deep = SYNC_DEEP
+      wide = SYNC_WIDE
+    else:
+      raise Exception('Illegal value of unconstrained_client option.')
+
     scenario['num_clients'] = 0  # use as many client as available.
-    # TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads
+    scenario['client_config']['outstanding_rpcs_per_channel'] = deep
-    # and that's probably too much (at least for wrapped languages).
+    scenario['client_config']['client_channels'] = wide
-    scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP
-    scenario['client_config']['client_channels'] = WIDE
     scenario['client_config']['async_client_threads'] = 0
   else:
     scenario['client_config']['outstanding_rpcs_per_channel'] = 1
     scenario['client_config']['client_channels'] = 1
     scenario['client_config']['async_client_threads'] = 1
 
+  if client_language:
+    # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
+    scenario['CLIENT_LANGUAGE'] = client_language
   if server_language:
     # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
     scenario['SERVER_LANGUAGE'] = server_language
@@ -196,27 +212,27 @@ class CXXLanguage:
       yield _ping_pong_scenario(
           'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure,
           categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure)
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_unconstrained_client=True, use_generic_payload=True,
+          unconstrained_client='async', use_generic_payload=True,
           secure=secure,
           categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_unconstrained_client=True, use_generic_payload=True,
+          unconstrained_client='async', use_generic_payload=True,
           server_core_limit=1, async_server_threads=1,
           secure=secure)
 
@@ -258,13 +274,13 @@ class CSharpLanguage:
     yield _ping_pong_scenario(
         'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        use_unconstrained_client=True,
+        unconstrained_client='async',
         categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        use_unconstrained_client=True)
+        unconstrained_client='async')
 
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@@ -277,6 +293,22 @@ class CSharpLanguage:
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
         server_language='c++', server_core_limit=1, async_server_threads=1)
 
+    yield _ping_pong_scenario(
+        'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        unconstrained_client='async', server_language='c++')
+
+    yield _ping_pong_scenario(
+        'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
+        unconstrained_client='sync', server_language='c++')
+
+    yield _ping_pong_scenario(
+        'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        unconstrained_client='async', client_language='c++')
+
+
   def __str__(self):
     return 'csharp'
 
@@ -313,14 +345,14 @@ class NodeLanguage:
     yield _ping_pong_scenario(
         'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        use_unconstrained_client=True,
+        unconstrained_client='async',
         categories=[SMOKETEST])
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
     #    'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
     #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-    #    use_unconstrained_client=True)
+    #    unconstrained_client='async')
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
@@ -369,19 +401,15 @@ class PythonLanguage:
         client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
         categories=[SMOKETEST])
 
-    # TODO(jtattermusch): 
+    yield _ping_pong_scenario(
-    # The qps_worker server gets thread starved with ~6400 threads, the GIL
+        'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
-    # enforces that a single thread runs at a time, with no way to set thread
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-    # priority.  Re-evaluate after changing DEEP and WIDE.
+        unconstrained_client='sync')
-    #yield _ping_pong_scenario(
-    #    'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
-    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-    #    use_unconstrained_client=True)
 
     yield _ping_pong_scenario(
         'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='SYNC_SERVER',
-        use_unconstrained_client=True)
+        unconstrained_client='async')
 
     yield _ping_pong_scenario(
         'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@@ -420,17 +448,15 @@ class RubyLanguage:
         client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
         categories=[SMOKETEST])
 
-    # TODO: scenario reports QPS of 0.0
+    yield _ping_pong_scenario(
-    #yield _ping_pong_scenario(
+        'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
-    #    'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        unconstrained_client='sync')
-    #    use_unconstrained_client=True)
 
-    # TODO: scenario reports QPS of 0.0
+    yield _ping_pong_scenario(
-    #yield _ping_pong_scenario(
+        'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
-    #    'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        unconstrained_client='sync')
-    #    use_unconstrained_client=True)
 
     yield _ping_pong_scenario(
         'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@@ -492,26 +518,26 @@ class JavaLanguage:
       yield _ping_pong_scenario(
           'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
           categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_unconstrained_client=True, use_generic_payload=True,
+          unconstrained_client='async', use_generic_payload=True,
           secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_unconstrained_client=True, use_generic_payload=True,
+          unconstrained_client='async', use_generic_payload=True,
           async_server_threads=1,
           secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
 
@@ -560,25 +586,28 @@ class GoLanguage:
           secure=secure,
           categories=smoketest_categories)
 
+      # unconstrained_client='async' is intended (client uses goroutines)
       yield _ping_pong_scenario(
           'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
           client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure,
           categories=smoketest_categories)
 
+      # unconstrained_client='async' is intended (client uses goroutines)
       yield _ping_pong_scenario(
           'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          use_unconstrained_client=True,
+          unconstrained_client='async',
           secure=secure)
 
+      # unconstrained_client='async' is intended (client uses goroutines)
       # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
       # but that's mostly because of lack of better name of the enum value.
       yield _ping_pong_scenario(
           'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_unconstrained_client=True, use_generic_payload=True,
+          unconstrained_client='async', use_generic_payload=True,
           secure=secure)
 
       # TODO(jtattermusch): add scenarios go vs C++ 

tools/run_tests/run_performance_tests.py

@@ -304,7 +304,11 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
           # 'SERVER_LANGUAGE' is an indicator for this script to pick
           # a server in different language.
           custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
+          custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None)
           scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
+          if custom_server_lang and custom_client_lang:
+            raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE'
+                            'in the same scenario')
           if custom_server_lang:
             if not workers_by_lang.get(custom_server_lang, []):
               print 'Warning: Skipping scenario %s as' % scenario_json['name']
@@ -314,6 +318,16 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
             for idx in range(0, scenario_json['num_servers']):
               # replace first X workers by workers of a different language
               workers[idx] = workers_by_lang[custom_server_lang][idx]
+          if custom_client_lang:
+            if not workers_by_lang.get(custom_client_lang, []):
+              print 'Warning: Skipping scenario %s as' % scenario_json['name']
+              print('CLIENT_LANGUAGE is set to %s yet the language has '
+                    'not been selected with -l' % custom_client_lang)
+              continue
+            for idx in range(scenario_json['num_servers'], len(workers)):
+              # replace all client workers by workers of a different language,
+              # leave num_server workers as they are server workers.
+              workers[idx] = workers_by_lang[custom_client_lang][idx]
           scenario = create_scenario_jobspec(scenario_json,
                                              workers,
                                              remote_host=remote_host,
@@ -360,8 +374,8 @@ argp.add_argument('--bq_result_table', default=None, type=str,
                   help='Bigquery "dataset.table" to upload results to.')
 argp.add_argument('--category',
                   choices=['smoketest','all'],
-                  default='smoketest',
+                  default='all',
-                  help='Select a category of tests to run. Smoketest runs by default.')
+                  help='Select a category of tests to run.')
 argp.add_argument('--netperf',
                   default=False,
                   action='store_const',