Merge pull request #6246 from jtattermusch/benchmarking_add_more_scenarios

Benchmarking: fine tune scenario configuration

tools/run_tests/performance/scenario_config.py

@@ -34,6 +34,9 @@ WARMUP_SECONDS=5
 JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
 BENCHMARK_SECONDS=30
 
+SECURE_SECARGS = {'use_test_ca': True,
+                  'server_host_override': 'foo.test.google.fr'}
+
 HISTOGRAM_PARAMS = {
   'resolution': 0.01,
   'max_possible': 60e9,
@@ -82,8 +85,7 @@ class CXXLanguage:
     for secure in [True, False]:
       if secure:
         secstr = 'secure'
-        secargs = {'use_test_ca': True,
+        secargs = SECURE_SECARGS
-                   'server_host_override': 'foo.test.google.fr'}
       else:
         secstr = 'insecure'
         secargs = None
@@ -325,8 +327,33 @@ class CSharpLanguage:
     return 100
 
   def scenarios(self):
-    # TODO(jtattermusch): add more scenarios
+    secargs = SECURE_SECARGS
-    secargs = None
+    yield {
+        'name': 'csharp_protobuf_async_streaming_qps_unconstrained',
+        'num_servers': 1,
+        'num_clients': 0,
+        'client_config': {
+          'client_type': 'ASYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': DEEP,
+          'client_channels': WIDE,
+          'async_client_threads': 0,
+          'rpc_type': 'STREAMING',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'ASYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 0,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
     yield {
         'name': 'csharp_generic_async_streaming_ping_pong',
         'num_servers': 1,
@@ -348,7 +375,7 @@ class CSharpLanguage:
           'server_type': 'ASYNC_GENERIC_SERVER',
           'security_params': secargs,
           'core_limit': 0,
-          'async_server_threads': 1,
+          'async_server_threads': 0,
           'payload_config': EMPTY_GENERIC_PAYLOAD,
         },
         'warmup_seconds': WARMUP_SECONDS,
@@ -375,7 +402,7 @@ class CSharpLanguage:
           'server_type': 'ASYNC_SERVER',
           'security_params': secargs,
           'core_limit': 0,
-          'async_server_threads': 1,
+          'async_server_threads': 0,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS
@@ -401,7 +428,7 @@ class CSharpLanguage:
           'server_type': 'ASYNC_SERVER',
           'security_params': secargs,
           'core_limit': 0,
-          'async_server_threads': 1,
+          'async_server_threads': 0,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS
@@ -426,7 +453,7 @@ class CSharpLanguage:
         'server_config': {
           'server_type': 'SYNC_SERVER',
           'security_params': secargs,
-          'core_limit': 0,
+          'core_limit': 1,
           'async_server_threads': 1,
         },
         'warmup_seconds': WARMUP_SECONDS,
@@ -452,7 +479,7 @@ class NodeLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
-    secargs = None
+    secargs = SECURE_SECARGS
     yield {
         'name': 'node_protobuf_unary_ping_pong',
         'num_servers': 1,
@@ -498,7 +525,7 @@ class RubyLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
-    secargs = None
+    secargs = SECURE_SECARGS
     yield {
         'name': 'ruby_protobuf_unary_ping_pong',
         'num_servers': 1,
@@ -544,33 +571,40 @@ class JavaLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
-    secargs = None
+    for secure in [True, False]:
-    yield {
+      if secure:
-        'name': 'java_protobuf_unary_ping_pong',
+        secstr = 'secure'
-        'num_servers': 1,
+        secargs = SECURE_SECARGS
-        'num_clients': 1,
+      else:
-        'client_config': {
+        secstr = 'insecure'
-          'client_type': 'SYNC_CLIENT',
+        secargs = None
-          'security_params': secargs,
+
-          'outstanding_rpcs_per_channel': 1,
+      yield {
-          'client_channels': 1,
+          'name': 'java_protobuf_unary_ping_pong_%s' % secstr,
-          'async_client_threads': 1,
+          'num_servers': 1,
-          'rpc_type': 'UNARY',
+          'num_clients': 1,
-          'load_params': {
+          'client_config': {
-            'closed_loop': {}
+            'client_type': 'SYNC_CLIENT',
+            'security_params': secargs,
+            'outstanding_rpcs_per_channel': 1,
+            'client_channels': 1,
+            'async_client_threads': 1,
+            'rpc_type': 'UNARY',
+            'load_params': {
+              'closed_loop': {}
+            },
+            'payload_config': EMPTY_PROTO_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'server_config': {
-          'histogram_params': HISTOGRAM_PARAMS,
+            'server_type': 'SYNC_SERVER',
-        },
+            'security_params': secargs,
-        'server_config': {
+            'core_limit': 0,
-          'server_type': 'SYNC_SERVER',
+            'async_server_threads': 1,
-          'security_params': secargs,
+          },
-          'core_limit': 0,
+          'warmup_seconds': JAVA_WARMUP_SECONDS,
-          'async_server_threads': 1,
+          'benchmark_seconds': BENCHMARK_SECONDS
-        },
+      }
-        'warmup_seconds': JAVA_WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
 
   def __str__(self):
     return 'java'