From 497966787a5955f03a4d7b2c0ee8d1948d7bd8e9 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 17 Oct 2016 10:01:37 -0700 Subject: [PATCH 1/3] fail performance tests if any jobs fail or timeout --- tools/run_tests/run_performance_tests.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 92149115fe2..5391ee6c923 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -353,10 +353,10 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', return scenarios - def finish_qps_workers(jobs): """Waits for given jobs to finish and eventually kills them.""" retries = 0 + num_killed = 0 while any(job.is_running() for job in jobs): for job in qpsworker_jobs: if job.is_running(): @@ -365,10 +365,11 @@ def finish_qps_workers(jobs): print('Killing all QPS workers.') for job in jobs: job.kill() + num_killed += 1 retries += 1 time.sleep(3) print('All QPS workers finished.') - + return num_killed argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', @@ -450,6 +451,7 @@ scenarios = create_scenarios(languages, if not scenarios: raise Exception('No scenarios to run') +num_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) @@ -457,8 +459,13 @@ for scenario in scenarios: try: for worker in scenario.workers: worker.start() - jobset.run([scenario.jobspec, - create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], - newline_on_success=True, maxjobs=1) + jobset_failures, _ = jobset.run([scenario.jobspec, + create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], + newline_on_success=True, maxjobs=1) + num_failures += jobset_failures finally: - finish_qps_workers(scenario.workers) + # Consider jobs that need to be killed as failures + num_failures += finish_qps_workers(scenario.workers) + +if num_failures > 0: + raise Exception('Failures occured') From cac93f673e80a4568790b6e478fbe7dd95962f34 Mon Sep 17 00:00:00 2001 From: Alex Polcyn Date: Wed, 19 Oct 2016 09:27:57 -0700 Subject: [PATCH 2/3] clean up error counting and reporting --- tools/run_tests/run_performance_tests.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 5391ee6c923..732e2675617 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -353,6 +353,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', return scenarios + def finish_qps_workers(jobs): """Waits for given jobs to finish and eventually kills them.""" retries = 0 @@ -451,7 +452,8 @@ scenarios = create_scenarios(languages, if not scenarios: raise Exception('No scenarios to run') -num_failures = 0 +total_scenario_failures = 0 +total_jobs_killed = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) @@ -459,13 +461,14 @@ for scenario in scenarios: try: for worker in scenario.workers: worker.start() - jobset_failures, _ = jobset.run([scenario.jobspec, + scenario_failures, _ = jobset.run([scenario.jobspec, create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], newline_on_success=True, maxjobs=1) - num_failures += jobset_failures + total_scenario_failures += scenario_failures finally: # Consider jobs that need to be killed as failures - num_failures += finish_qps_workers(scenario.workers) + total_jobs_killed += finish_qps_workers(scenario.workers) if num_failures > 0: - raise Exception('Failures occured') + print(str(total_scenario_failures) + " scenarios failed and " + str(total_jobs_killed) + " jobs killed") + sys.exit(1) From 898a2e91f956fe36e6061e3b20727e352bccb263 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Sat, 22 Oct 2016 17:41:23 -0700 Subject: [PATCH 3/3] clean up error message --- tools/run_tests/run_performance_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 732e2675617..317656b40d2 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -453,7 +453,7 @@ if not scenarios: raise Exception('No scenarios to run') total_scenario_failures = 0 -total_jobs_killed = 0 +qps_workers_killed = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) @@ -466,9 +466,9 @@ for scenario in scenarios: newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures finally: - # Consider jobs that need to be killed as failures - total_jobs_killed += finish_qps_workers(scenario.workers) + # Consider qps workers that need to be killed as failures + qps_workers_killed += finish_qps_workers(scenario.workers) -if num_failures > 0: - print(str(total_scenario_failures) + " scenarios failed and " + str(total_jobs_killed) + " jobs killed") +if total_scenario_failures > 0 or qps_workers_killed > 0: + print ("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed)) sys.exit(1)