From 9992bdb74ec2a39b7d8cd867ee4f3b3f96d151bb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 6 Sep 2017 15:00:31 -0700 Subject: [PATCH 1/5] Query CPU cost of tests and feed that into test runner --- tools/run_tests/run_tests.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4311e538397..6bb61c0b70a 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -69,17 +69,20 @@ _POLLING_STRATEGIES = { } -def get_flaky_tests(limit=None): +BigQueryTestData = collections.namedtuple('BigQueryTestData', 'name flaky cpu') + + +def get_bqtest_data(limit=None): import big_query_utils bq = big_query_utils.create_big_query() query = """ SELECT - filtered_test_name, + filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, AVG(cpu_measured) as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, - result + result, cpu_measured FROM [grpc-testing:jenkins_test_results.aggregate_results] WHERE @@ -89,15 +92,15 @@ SELECT GROUP BY filtered_test_name HAVING - SUM(result != 'PASSED' AND result != 'SKIPPED') > 0""" + flaky OR cpu > 0""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) page = bq.jobs().getQueryResults( pageToken=None, **query_job['jobReference']).execute(num_retries=3) - flake_names = [row['f'][0]['v'] for row in page['rows']] - return flake_names + test_data = [BigQueryTestData(row['f'][0]['v'], row['f'][1]['v'] == 'true', float(row['f'][2]['v'])) for row in page['rows']] + return test_data def platform_string(): @@ -141,6 +144,9 @@ class Config(object): if not flaky and shortname and shortname in flaky_tests: print('Setting %s to flaky' % shortname) flaky = True + if shortname in test_times: + print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, test_times[shortname])) + cpu_cost = test_times[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1254,9 +1260,12 @@ argp.add_argument('--disable_auto_set_flakes', default=False, const=True, action args = argp.parse_args() flaky_tests = set() +test_times = {} if not args.disable_auto_set_flakes: try: - flaky_tests = set(get_flaky_tests()) + for test in get_bqtest_data(): + if test.flaky: flaky_tests.add(test.name) + if test.cpu > 0: test_times[test.name] = test.cpu except: print("Unexpected error getting flaky tests:", sys.exc_info()[0]) From 120d4fda0715b157ed3a19d2a557906dac29f4b0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 12:25:25 -0700 Subject: [PATCH 2/5] Use max cpu_measured instead of average (for safety) --- tools/run_tests/run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 6bb61c0b70a..3edd28fd5e9 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -78,7 +78,9 @@ def get_bqtest_data(limit=None): bq = big_query_utils.create_big_query() query = """ SELECT - filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, AVG(cpu_measured) as cpu + filtered_test_name, + SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, + MAX(cpu_measured) as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, From 0b86d03611204ef3b4f445c489eb4d8f041f6479 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 13:47:45 -0700 Subject: [PATCH 3/5] Better variable name --- tools/run_tests/run_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 6bb61c0b70a..8b3c629d5fd 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -144,9 +144,9 @@ class Config(object): if not flaky and shortname and shortname in flaky_tests: print('Setting %s to flaky' % shortname) flaky = True - if shortname in test_times: - print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, test_times[shortname])) - cpu_cost = test_times[shortname] + if shortname in shortname_to_cpu: + print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, shortname_to_cpu[shortname])) + cpu_cost = shortname_to_cpu[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1260,12 +1260,12 @@ argp.add_argument('--disable_auto_set_flakes', default=False, const=True, action args = argp.parse_args() flaky_tests = set() -test_times = {} +shortname_to_cpu = {} if not args.disable_auto_set_flakes: try: for test in get_bqtest_data(): if test.flaky: flaky_tests.add(test.name) - if test.cpu > 0: test_times[test.name] = test.cpu + if test.cpu > 0: shortname_to_cpu[test.name] = test.cpu except: print("Unexpected error getting flaky tests:", sys.exc_info()[0]) From 8d0fef20890d6bddc3739e5648910decfb0dabd5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 13:59:47 -0700 Subject: [PATCH 4/5] Sort tests by cpu usage to better bin-pack --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 72ded5694af..b66c5f7f71f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1527,7 +1527,7 @@ def _build_and_run( # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis and args.max_time <= 0: - massaged_one_run = sorted(one_run, key=lambda x: x.shortname) + massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. From b9f99f0de2ac5ff08de722be6f3baa37af42bc77 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 7 Sep 2017 15:55:39 -0700 Subject: [PATCH 5/5] Fix use of grpc_channel_filter from concurrent merges --- test/core/channel/channel_stack_builder_test.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/core/channel/channel_stack_builder_test.c b/test/core/channel/channel_stack_builder_test.c index be6afb7c07e..682efd14385 100644 --- a/test/core/channel/channel_stack_builder_test.c +++ b/test/core/channel/channel_stack_builder_test.c @@ -59,10 +59,6 @@ static void channel_func(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, GRPC_CLOSURE_SCHED(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); } -static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - return gpr_strdup("peer"); -} - bool g_replacement_fn_called = false; bool g_original_fn_called = false; void set_arg_once_fn(grpc_channel_stack *channel_stack, @@ -94,7 +90,6 @@ const grpc_channel_filter replacement_filter = { 0, channel_init_func, channel_destroy_func, - get_peer, grpc_channel_next_get_info, "filter_name"}; @@ -108,7 +103,6 @@ const grpc_channel_filter original_filter = { 0, channel_init_func, channel_destroy_func, - get_peer, grpc_channel_next_get_info, "filter_name"};