Merge pull request #8202 from ctiller/perf-wha

Be explicit as to which tests to run
pull/8258/merge
Craig Tiller 8 years ago committed by GitHub
commit c9742d37ac
  1. 3
      tools/jenkins/run_full_performance.sh
  2. 26
      tools/run_tests/performance/scenario_config.py
  3. 45
      tools/run_tests/run_performance_tests.py

@ -38,7 +38,7 @@ cd $(dirname $0)/../..
tools/run_tests/run_performance_tests.py \
-l c++ csharp node ruby java python go \
--netperf \
--category all \
--category scalable \
--bq_result_table performance_test.performance_experiment \
--remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \
|| EXIT_CODE=1
@ -53,4 +53,3 @@ tools/run_tests/run_performance_tests.py \
|| EXIT_CODE=1
exit $EXIT_CODE

@ -190,7 +190,7 @@ class CXXLanguage:
# TODO(ctiller): add 70% load latency test
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = [SMOKETEST] if secure else []
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
yield _ping_pong_scenario(
'cpp_generic_async_streaming_ping_pong_%s' % secstr,
@ -270,7 +270,7 @@ class CSharpLanguage:
'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING',
client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True,
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
@ -279,7 +279,7 @@ class CSharpLanguage:
yield _ping_pong_scenario(
'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY',
@ -301,7 +301,7 @@ class CSharpLanguage:
'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1,
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
@ -358,13 +358,13 @@ class NodeLanguage:
yield _ping_pong_scenario(
'node_protobuf_unary_ping_pong', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
categories=[SMOKETEST])
categories=[SCALABLE, SMOKETEST])
yield _ping_pong_scenario(
'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
unconstrained_client='async',
categories=[SMOKETEST])
categories=[SCALABLE, SMOKETEST])
# TODO(jtattermusch): make this scenario work
#yield _ping_pong_scenario(
@ -403,7 +403,7 @@ class PythonLanguage:
'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
use_generic_payload=True,
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
@ -416,7 +416,7 @@ class PythonLanguage:
yield _ping_pong_scenario(
'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
@ -432,7 +432,7 @@ class PythonLanguage:
'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
server_language='c++', server_core_limit=1, async_server_threads=1,
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
@ -458,12 +458,12 @@ class RubyLanguage:
yield _ping_pong_scenario(
'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'ruby_protobuf_unary_ping_pong', rpc_type='UNARY',
client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
categories=[SMOKETEST])
categories=[SMOKETEST, SCALABLE])
yield _ping_pong_scenario(
'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
@ -504,7 +504,7 @@ class JavaLanguage:
def scenarios(self):
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = [SMOKETEST] if secure else []
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
yield _ping_pong_scenario(
'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
@ -581,7 +581,7 @@ class GoLanguage:
def scenarios(self):
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = [SMOKETEST] if secure else []
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
# ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
# but that's mostly because of lack of better name of the enum value.

@ -280,7 +280,7 @@ def create_qpsworkers(languages, worker_hosts):
for worker_idx, worker in enumerate(workers)]
Scenario = collections.namedtuple('Scenario', 'jobspec workers')
Scenario = collections.namedtuple('Scenario', 'jobspec workers name')
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
@ -307,7 +307,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
create_netperf_jobspec(server_host=netperf_server,
client_host=netperf_client,
bq_result_table=bq_result_table),
_NO_WORKERS))
_NO_WORKERS, 'netperf'))
for language in languages:
for scenario_json in language.scenarios():
@ -347,7 +347,8 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
[w.host_and_port for w in workers],
remote_host=remote_host,
bq_result_table=bq_result_table),
workers)
workers,
scenario_json['name'])
scenarios.append(scenario)
return scenarios
@ -382,6 +383,11 @@ argp.add_argument('--remote_worker_host',
nargs='+',
default=[],
help='Worker hosts where to start QPS workers.')
argp.add_argument('--dry_run',
default=False,
action='store_const',
const=True,
help='Just list scenarios to be run, but don\'t run them.')
argp.add_argument('-r', '--regex', default='.*', type=str,
help='Regex to select scenarios to run.')
argp.add_argument('--bq_result_table', default=None, type=str,
@ -412,16 +418,18 @@ if args.remote_worker_host:
if args.remote_driver_host:
remote_hosts.add(args.remote_driver_host)
if remote_hosts:
archive_repo(languages=[str(l) for l in languages])
prepare_remote_hosts(remote_hosts, prepare_local=True)
else:
prepare_remote_hosts([], prepare_local=True)
if not args.dry_run:
if remote_hosts:
archive_repo(languages=[str(l) for l in languages])
prepare_remote_hosts(remote_hosts, prepare_local=True)
else:
prepare_remote_hosts([], prepare_local=True)
build_local = False
if not args.remote_driver_host:
build_local = True
build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local)
if not args.dry_run:
build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local)
qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host)
@ -443,11 +451,14 @@ if not scenarios:
raise Exception('No scenarios to run')
for scenario in scenarios:
try:
for worker in scenario.workers:
worker.start()
jobset.run([scenario.jobspec,
create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
newline_on_success=True, maxjobs=1)
finally:
finish_qps_workers(scenario.workers)
if args.dry_run:
print(scenario.name)
else:
try:
for worker in scenario.workers:
worker.start()
jobset.run([scenario.jobspec,
create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
newline_on_success=True, maxjobs=1)
finally:
finish_qps_workers(scenario.workers)

Loading…
Cancel
Save