mark some scenarios as smoketest and add a cmdline flag

tools/run_tests/performance/scenario_config.py

@@ -34,6 +34,8 @@ WARMUP_SECONDS=5
 JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
 BENCHMARK_SECONDS=30
 
+SMOKETEST='smoketest'
+
 SECURE_SECARGS = {'use_test_ca': True,
                   'server_host_override': 'foo.test.google.fr'}
 
@@ -164,12 +166,14 @@ class CXXLanguage:
     # TODO(ctiller): add 70% load latency test
     for secure in [True, False]:
       secstr = 'secure' if secure else 'insecure'
+      smoketest_categories = [SMOKETEST] if secure else None
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           use_generic_payload=True, server_core_limit=1, async_server_threads=1,
-          secure=secure)
+          secure=secure,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
@@ -181,7 +185,8 @@ class CXXLanguage:
           'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           server_core_limit=1, async_server_threads=1,
-          secure=secure)
+          secure=secure,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
@@ -194,7 +199,8 @@ class CXXLanguage:
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           server_core_limit=SINGLE_MACHINE_CORES/2,
           use_unconstrained_client=True,
-          secure=secure)
+          secure=secure,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
@@ -208,7 +214,8 @@ class CXXLanguage:
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           use_unconstrained_client=True, use_generic_payload=True,
           server_core_limit=SINGLE_MACHINE_CORES/2,
-          secure=secure)
+          secure=secure,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
@@ -236,7 +243,8 @@ class CSharpLanguage:
     yield _ping_pong_scenario(
         'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        use_generic_payload=True)
+        use_generic_payload=True,
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
@@ -244,7 +252,8 @@ class CSharpLanguage:
 
     yield _ping_pong_scenario(
         'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY',
@@ -253,7 +262,8 @@ class CSharpLanguage:
     yield _ping_pong_scenario(
         'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        use_unconstrained_client=True)
+        use_unconstrained_client=True,
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
@@ -263,7 +273,8 @@ class CSharpLanguage:
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
         client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', server_core_limit=1, async_server_threads=1)
+        server_language='c++', server_core_limit=1, async_server_threads=1,
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
@@ -300,12 +311,14 @@ class NodeLanguage:
 
     yield _ping_pong_scenario(
         'node_protobuf_unary_ping_pong', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        use_unconstrained_client=True)
+        use_unconstrained_client=True,
+        categories=[SMOKETEST])
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
@@ -344,7 +357,8 @@ class PythonLanguage:
     yield _ping_pong_scenario(
         'python_generic_async_streaming_ping_pong', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        use_generic_payload=True)
+        use_generic_payload=True,
+        categories=[SMOKETEST])
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
@@ -358,7 +372,8 @@ class PythonLanguage:
 
     yield _ping_pong_scenario(
         'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER')
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
@@ -375,7 +390,8 @@ class PythonLanguage:
     yield _ping_pong_scenario(
         'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
         client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', server_core_limit=1, async_server_threads=1)
+        server_language='c++', server_core_limit=1, async_server_threads=1,
+        categories=[SMOKETEST])
 
     # TODO(jtattermusch): make this scenario work
     #yield _ping_pong_scenario(
@@ -401,11 +417,13 @@ class RubyLanguage:
   def scenarios(self):
     yield _ping_pong_scenario(
         'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER')
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
 
     yield _ping_pong_scenario(
         'ruby_protobuf_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER')
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
 
     # TODO: scenario reports QPS of 0.0
     #yield _ping_pong_scenario(
@@ -448,12 +466,14 @@ class JavaLanguage:
   def scenarios(self):
     for secure in [True, False]:
       secstr = 'secure' if secure else 'insecure'
+      smoketest_categories = [SMOKETEST] if secure else None
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           use_generic_payload=True, async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
@@ -465,7 +485,8 @@ class JavaLanguage:
           'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
@@ -477,7 +498,8 @@ class JavaLanguage:
           'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           use_unconstrained_client=True,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
 
       yield _ping_pong_scenario(
           'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',

tools/run_tests/run_performance_tests.py

@@ -244,7 +244,7 @@ def start_qpsworkers(languages, worker_hosts):
 
 
 def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
-                     bq_result_table=None):
+                     category='all', bq_result_table=None):
   """Create jobspecs for scenarios to run."""
   all_workers = [worker
                  for workers in workers_by_lang.values()
@@ -253,25 +253,26 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
   for language in languages:
     for scenario_json in language.scenarios():
       if re.search(args.regex, scenario_json['name']):
-        workers = workers_by_lang[str(language)]
+        if category in scenario_json.get('CATEGORIES', []) or category == 'all':
-        # 'SERVER_LANGUAGE' is an indicator for this script to pick
+          workers = workers_by_lang[str(language)]
-        # a server in different language.
+          # 'SERVER_LANGUAGE' is an indicator for this script to pick
-        custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
+          # a server in different language.
-        scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
+          custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
-        if custom_server_lang:
+          scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
-          if not workers_by_lang.get(custom_server_lang, []):
+          if custom_server_lang:
-            print 'Warning: Skipping scenario %s as' % scenario_json['name']
+            if not workers_by_lang.get(custom_server_lang, []):
-            print('SERVER_LANGUAGE is set to %s yet the language has '
+              print 'Warning: Skipping scenario %s as' % scenario_json['name']
-                  'not been selected with -l' % custom_server_lang)
+              print('SERVER_LANGUAGE is set to %s yet the language has '
-            continue
+                    'not been selected with -l' % custom_server_lang)
-          for idx in range(0, scenario_json['num_servers']):
+              continue
-            # replace first X workers by workers of a different language
+            for idx in range(0, scenario_json['num_servers']):
-            workers[idx] = workers_by_lang[custom_server_lang][idx]
+              # replace first X workers by workers of a different language
-        scenario = create_scenario_jobspec(scenario_json,
+              workers[idx] = workers_by_lang[custom_server_lang][idx]
-                                           workers,
+          scenario = create_scenario_jobspec(scenario_json,
-                                           remote_host=remote_host,
+                                             workers,
-                                           bq_result_table=bq_result_table)
+                                             remote_host=remote_host,
-        scenarios.append(scenario)
+                                             bq_result_table=bq_result_table)
+          scenarios.append(scenario)
 
   # the very last scenario requests shutting down the workers.
   scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
@@ -298,7 +299,7 @@ argp = argparse.ArgumentParser(description='Run performance tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(scenario_config.LANGUAGES.keys()),
                   nargs='+',
-                  default=['all'],
+                  required=True,
                   help='Languages to benchmark.')
 argp.add_argument('--remote_driver_host',
                   default=None,
@@ -311,6 +312,10 @@ argp.add_argument('-r', '--regex', default='.*', type=str,
                   help='Regex to select scenarios to run.')
 argp.add_argument('--bq_result_table', default=None, type=str,
                   help='Bigquery "dataset.table" to upload results to.')
+argp.add_argument('--category',
+                  choices=['smoketest','all'],
+                  default='smoketest',
+                  help='Select a category of tests to run. Smoketest runs by default.')
 
 args = argp.parse_args()
 
@@ -354,6 +359,7 @@ try:
                                workers_by_lang=worker_addresses,
                                remote_host=args.remote_driver_host,
                                regex=args.regex,
+                               category=args.category,
                                bq_result_table=args.bq_result_table)
   if not scenarios:
     raise Exception('No scenarios to run')