From d439b4e2d9d29009643be75c4110ebe970694998 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 17:57:18 -0700 Subject: [PATCH 1/5] Split the benchmarks into individual jobspecs --- build.yaml | 14 ++++++++++++++ tools/run_tests/run_tests.py | 18 ++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/build.yaml b/build.yaml index d15cf0339ca..23c23b3bc8c 100644 --- a/build.yaml +++ b/build.yaml @@ -3466,6 +3466,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3487,6 +3488,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3508,6 +3510,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3529,6 +3532,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3550,6 +3554,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3571,6 +3576,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3592,6 +3598,7 @@ targets: - gpr args: - --benchmark_min_time=4 + benchmark: true defaults: benchmark platforms: - mac @@ -3613,6 +3620,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3636,6 +3644,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3663,6 +3672,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3689,6 +3699,7 @@ targets: - grpc++_test_config args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3716,6 +3727,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3741,6 +3753,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3762,6 +3775,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b38108d456c..784d68f2e3a 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -323,13 +323,19 @@ class CLanguage(object): if cpu_cost == 'capacity': cpu_cost = multiprocessing.cpu_count() if os.path.isfile(binary): + test_prefix = None if 'gtest' in target and target['gtest']: - # here we parse the output of --gtest_list_tests to build up a - # complete list of the tests contained in a binary - # for each test, we then add a job to run, filtering for just that - # test + test_prefix = 'gtest' + elif 'benchmark' in target and target['benchmark']: + test_prefix = 'benchmark' + + if test_prefix: + # here we parse the output of --gtest_list_tests (or + # --benchmark_list_tests)to build up a complete list of + # the tests contained in a binary for each test, we then + # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, '--gtest_list_tests'], + tests = subprocess.check_output([binary, '--%s_list_tests' % test_prefix], stderr=fnull) base = None for line in tests.split('\n'): @@ -342,7 +348,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, '--gtest_filter=%s' % test] + target['args'] + cmdline = [binary, '--%s_filter=%s' % (test_prefix, test)] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, From 7839d82f28f2fad5a54616e36636ffee825b641c Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 12 Sep 2017 09:46:22 -0700 Subject: [PATCH 2/5] Address github comments --- tools/run_tests/run_tests.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 784d68f2e3a..39f492d90c5 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -323,19 +323,26 @@ class CLanguage(object): if cpu_cost == 'capacity': cpu_cost = multiprocessing.cpu_count() if os.path.isfile(binary): - test_prefix = None + list_test_command = None + filter_test_command = None + + # these are the flag defined by gtest and benchmark framework to list + # and filter test runs. We use them to split each individual test + # into its own JobSpec, and thus into its own process. if 'gtest' in target and target['gtest']: - test_prefix = 'gtest' + list_test_command = '--gtest_list_tests' + filter_test_command = '--gtest_filter=%s' elif 'benchmark' in target and target['benchmark']: - test_prefix = 'benchmark' + list_test_command = '--benchmark_list_tests' + filter_test_command = '--benchmark_filter=%s' - if test_prefix: + if list_test_command: # here we parse the output of --gtest_list_tests (or # --benchmark_list_tests)to build up a complete list of # the tests contained in a binary for each test, we then # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, '--%s_list_tests' % test_prefix], + tests = subprocess.check_output([binary, list_test_command], stderr=fnull) base = None for line in tests.split('\n'): @@ -348,7 +355,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, '--%s_filter=%s' % (test_prefix, test)] + target['args'] + cmdline = [binary, filter_test_command % test] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, From 128bbc7e80dc51881a9fb39eade77df259b4e938 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 20 Sep 2017 14:14:29 -0400 Subject: [PATCH 3/5] Check for benchmark before check for test --- tools/run_tests/run_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 39f492d90c5..1cbe09bb920 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -329,12 +329,12 @@ class CLanguage(object): # these are the flag defined by gtest and benchmark framework to list # and filter test runs. We use them to split each individual test # into its own JobSpec, and thus into its own process. - if 'gtest' in target and target['gtest']: - list_test_command = '--gtest_list_tests' - filter_test_command = '--gtest_filter=%s' - elif 'benchmark' in target and target['benchmark']: + if 'benchmark' in target and target['benchmark']: list_test_command = '--benchmark_list_tests' filter_test_command = '--benchmark_filter=%s' + elif 'gtest' in target and target['gtest']: + list_test_command = '--gtest_list_tests' + filter_test_command = '--gtest_filter=%s' if list_test_command: # here we parse the output of --gtest_list_tests (or From 5dda3d3e2229ced8ec05f0d2ba121c42f6e77186 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 20 Sep 2017 15:54:36 -0400 Subject: [PATCH 4/5] Upate test.json template --- .../run_tests/generated/tests.json.template | 1 + tools/run_tests/generated/tests.json | 201 ++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/templates/tools/run_tests/generated/tests.json.template b/templates/tools/run_tests/generated/tests.json.template index 10ab2e445a4..9f82824a0ac 100644 --- a/templates/tools/run_tests/generated/tests.json.template +++ b/templates/tools/run_tests/generated/tests.json.template @@ -9,6 +9,7 @@ "platforms": tgt.platforms, "ci_platforms": tgt.ci_platforms, "gtest": tgt.gtest, + "benchmark": tgt.get("benchmark", False), "exclude_configs": tgt.get("exclude_configs", []), "exclude_iomgrs": tgt.get("exclude_iomgrs", []), "args": tgt.get("args", []), diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 72dfbc0b982..59c73ed8f43 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3,6 +3,7 @@ [ { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -25,6 +26,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -47,6 +49,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -69,6 +72,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -91,6 +95,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -113,6 +118,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -137,6 +143,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -159,6 +166,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -181,6 +189,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -203,6 +212,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -225,6 +235,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -247,6 +258,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -269,6 +281,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -291,6 +304,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -313,6 +327,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -335,6 +350,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -357,6 +373,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -379,6 +396,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -401,6 +419,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -423,6 +442,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -445,6 +465,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -469,6 +490,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -491,6 +513,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -515,6 +538,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -537,6 +561,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -559,6 +584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -583,6 +609,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -605,6 +632,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -623,6 +651,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -645,6 +674,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -665,6 +695,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -687,6 +718,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -709,6 +741,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -729,6 +762,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -749,6 +783,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -771,6 +806,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -793,6 +829,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -815,6 +852,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -837,6 +875,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -859,6 +898,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -881,6 +921,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -903,6 +944,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -925,6 +967,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -947,6 +990,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -969,6 +1013,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -991,6 +1036,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1013,6 +1059,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1035,6 +1082,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1057,6 +1105,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1079,6 +1128,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1101,6 +1151,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1123,6 +1174,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1145,6 +1197,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1167,6 +1220,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1189,6 +1243,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1211,6 +1266,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1233,6 +1289,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1255,6 +1312,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1277,6 +1335,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1299,6 +1358,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1323,6 +1383,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1345,6 +1406,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1367,6 +1429,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1387,6 +1450,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1409,6 +1473,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1431,6 +1496,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1449,6 +1515,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1467,6 +1534,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1489,6 +1557,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1511,6 +1580,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1533,6 +1603,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1555,6 +1626,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1575,6 +1647,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1591,6 +1664,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1613,6 +1687,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1635,6 +1710,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1657,6 +1733,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1679,6 +1756,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1701,6 +1779,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1723,6 +1802,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1745,6 +1825,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1765,6 +1846,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1787,6 +1869,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1809,6 +1892,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1831,6 +1915,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1853,6 +1938,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1875,6 +1961,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1897,6 +1984,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1921,6 +2009,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1943,6 +2032,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1965,6 +2055,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1983,6 +2074,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2005,6 +2097,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2027,6 +2120,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2049,6 +2143,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2071,6 +2166,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2095,6 +2191,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2119,6 +2216,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2141,6 +2239,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2163,6 +2262,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2185,6 +2285,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2207,6 +2308,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2229,6 +2331,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2251,6 +2354,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2273,6 +2377,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2295,6 +2400,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2317,6 +2423,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2337,6 +2444,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2359,6 +2467,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2381,6 +2490,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2403,6 +2513,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2425,6 +2536,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2449,6 +2561,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2471,6 +2584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2493,6 +2607,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2517,6 +2632,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2539,6 +2655,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2561,6 +2678,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2585,6 +2703,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2609,6 +2728,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2631,6 +2751,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2653,6 +2774,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2675,6 +2797,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2695,6 +2818,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2717,6 +2841,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2739,6 +2864,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2761,6 +2887,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2783,6 +2910,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2805,6 +2933,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2829,6 +2958,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2851,6 +2981,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2873,6 +3004,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2895,6 +3027,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2917,6 +3050,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2939,6 +3073,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2961,6 +3096,7 @@ "args": [ "--benchmark_min_time=4" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2983,6 +3119,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3005,6 +3142,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3032,6 +3170,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3059,6 +3198,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3086,6 +3226,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3113,6 +3254,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3135,6 +3277,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3155,6 +3298,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3177,6 +3321,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3199,6 +3344,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3221,6 +3367,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3241,6 +3388,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3267,6 +3415,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3289,6 +3438,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3311,6 +3461,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3333,6 +3484,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3355,6 +3507,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3377,6 +3530,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3399,6 +3553,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3421,6 +3576,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3443,6 +3599,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3465,6 +3622,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3487,6 +3645,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3511,6 +3670,7 @@ "args": [ "--generated_file_path=gens/src/proto/grpc/testing/" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3533,6 +3693,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3555,6 +3716,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3577,6 +3739,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3603,6 +3766,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3629,6 +3793,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3651,6 +3816,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3673,6 +3839,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3693,6 +3860,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3715,6 +3883,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3737,6 +3906,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3759,6 +3929,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3781,6 +3952,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3803,6 +3975,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3823,6 +3996,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3845,6 +4019,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3865,6 +4040,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3887,6 +4063,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3909,6 +4086,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3931,6 +4109,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3951,6 +4130,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3973,6 +4153,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3995,6 +4176,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4017,6 +4199,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4039,6 +4222,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4059,6 +4243,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4081,6 +4266,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4104,6 +4290,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4124,6 +4311,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4146,6 +4334,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4170,6 +4359,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4194,6 +4384,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4218,6 +4409,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4242,6 +4434,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4266,6 +4459,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4290,6 +4484,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4314,6 +4509,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4338,6 +4534,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4362,6 +4559,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4386,6 +4584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4409,6 +4608,7 @@ "--test_bin_name=resolver_component_test_unsecure", "--running_under_bazel=false" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4432,6 +4632,7 @@ "--test_bin_name=resolver_component_test", "--running_under_bazel=false" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", From 77013000e7fb7740a6e6f7cf76eb47f8a282baf2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 16 Oct 2017 16:11:37 -0700 Subject: [PATCH 5/5] Working benchmark parallelization --- tools/run_tests/run_tests.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 340e848e7f8..588784353a9 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -149,10 +149,8 @@ class Config(object): for k, v in environ.items(): actual_environ[k] = v if not flaky and shortname and shortname in flaky_tests: - print('Setting %s to flaky' % shortname) flaky = True if shortname in shortname_to_cpu: - print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, shortname_to_cpu[shortname])) cpu_cost = shortname_to_cpu[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, @@ -339,19 +337,24 @@ class CLanguage(object): # and filter test runs. We use them to split each individual test # into its own JobSpec, and thus into its own process. if 'benchmark' in target and target['benchmark']: - list_test_command = '--benchmark_list_tests' - filter_test_command = '--benchmark_filter=%s' + with open(os.devnull, 'w') as fnull: + tests = subprocess.check_output([binary, '--benchmark_list_tests'], + stderr=fnull) + base = None + for line in tests.split('\n'): + test = line.strip() + cmdline = [binary, '--benchmark_filter=%s$' % test] + target['args'] + out.append(self.config.job_spec(cmdline, + shortname='%s:%s %s' % (binary, test, shortname_ext), + cpu_cost=cpu_cost, + timeout_seconds=_DEFAULT_TIMEOUT_SECONDS * timeout_scaling, + environ=env)) elif 'gtest' in target and target['gtest']: - list_test_command = '--gtest_list_tests' - filter_test_command = '--gtest_filter=%s' - - if list_test_command: - # here we parse the output of --gtest_list_tests (or - # --benchmark_list_tests)to build up a complete list of - # the tests contained in a binary for each test, we then + # here we parse the output of --gtest_list_tests to build up a complete + # list of the tests contained in a binary for each test, we then # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, list_test_command], + tests = subprocess.check_output([binary, '--gtest_list_tests'], stderr=fnull) base = None for line in tests.split('\n'): @@ -364,7 +367,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, filter_test_command % test] + target['args'] + cmdline = [binary, '--gtest_filter=%s' % test] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost,