From d439b4e2d9d29009643be75c4110ebe970694998 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 17:57:18 -0700 Subject: [PATCH 01/12] Split the benchmarks into individual jobspecs --- build.yaml | 14 ++++++++++++++ tools/run_tests/run_tests.py | 18 ++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/build.yaml b/build.yaml index d15cf0339ca..23c23b3bc8c 100644 --- a/build.yaml +++ b/build.yaml @@ -3466,6 +3466,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3487,6 +3488,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3508,6 +3510,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3529,6 +3532,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3550,6 +3554,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3571,6 +3576,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3592,6 +3598,7 @@ targets: - gpr args: - --benchmark_min_time=4 + benchmark: true defaults: benchmark platforms: - mac @@ -3613,6 +3620,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3636,6 +3644,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3663,6 +3672,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3689,6 +3699,7 @@ targets: - grpc++_test_config args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3716,6 +3727,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark excluded_poll_engines: - poll @@ -3741,6 +3753,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac @@ -3762,6 +3775,7 @@ targets: - gpr args: - --benchmark_min_time=0 + benchmark: true defaults: benchmark platforms: - mac diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b38108d456c..784d68f2e3a 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -323,13 +323,19 @@ class CLanguage(object): if cpu_cost == 'capacity': cpu_cost = multiprocessing.cpu_count() if os.path.isfile(binary): + test_prefix = None if 'gtest' in target and target['gtest']: - # here we parse the output of --gtest_list_tests to build up a - # complete list of the tests contained in a binary - # for each test, we then add a job to run, filtering for just that - # test + test_prefix = 'gtest' + elif 'benchmark' in target and target['benchmark']: + test_prefix = 'benchmark' + + if test_prefix: + # here we parse the output of --gtest_list_tests (or + # --benchmark_list_tests)to build up a complete list of + # the tests contained in a binary for each test, we then + # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, '--gtest_list_tests'], + tests = subprocess.check_output([binary, '--%s_list_tests' % test_prefix], stderr=fnull) base = None for line in tests.split('\n'): @@ -342,7 +348,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, '--gtest_filter=%s' % test] + target['args'] + cmdline = [binary, '--%s_filter=%s' % (test_prefix, test)] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, From 7839d82f28f2fad5a54616e36636ffee825b641c Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 12 Sep 2017 09:46:22 -0700 Subject: [PATCH 02/12] Address github comments --- tools/run_tests/run_tests.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 784d68f2e3a..39f492d90c5 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -323,19 +323,26 @@ class CLanguage(object): if cpu_cost == 'capacity': cpu_cost = multiprocessing.cpu_count() if os.path.isfile(binary): - test_prefix = None + list_test_command = None + filter_test_command = None + + # these are the flag defined by gtest and benchmark framework to list + # and filter test runs. We use them to split each individual test + # into its own JobSpec, and thus into its own process. if 'gtest' in target and target['gtest']: - test_prefix = 'gtest' + list_test_command = '--gtest_list_tests' + filter_test_command = '--gtest_filter=%s' elif 'benchmark' in target and target['benchmark']: - test_prefix = 'benchmark' + list_test_command = '--benchmark_list_tests' + filter_test_command = '--benchmark_filter=%s' - if test_prefix: + if list_test_command: # here we parse the output of --gtest_list_tests (or # --benchmark_list_tests)to build up a complete list of # the tests contained in a binary for each test, we then # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, '--%s_list_tests' % test_prefix], + tests = subprocess.check_output([binary, list_test_command], stderr=fnull) base = None for line in tests.split('\n'): @@ -348,7 +355,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, '--%s_filter=%s' % (test_prefix, test)] + target['args'] + cmdline = [binary, filter_test_command % test] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, From 128bbc7e80dc51881a9fb39eade77df259b4e938 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 20 Sep 2017 14:14:29 -0400 Subject: [PATCH 03/12] Check for benchmark before check for test --- tools/run_tests/run_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 39f492d90c5..1cbe09bb920 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -329,12 +329,12 @@ class CLanguage(object): # these are the flag defined by gtest and benchmark framework to list # and filter test runs. We use them to split each individual test # into its own JobSpec, and thus into its own process. - if 'gtest' in target and target['gtest']: - list_test_command = '--gtest_list_tests' - filter_test_command = '--gtest_filter=%s' - elif 'benchmark' in target and target['benchmark']: + if 'benchmark' in target and target['benchmark']: list_test_command = '--benchmark_list_tests' filter_test_command = '--benchmark_filter=%s' + elif 'gtest' in target and target['gtest']: + list_test_command = '--gtest_list_tests' + filter_test_command = '--gtest_filter=%s' if list_test_command: # here we parse the output of --gtest_list_tests (or From 5dda3d3e2229ced8ec05f0d2ba121c42f6e77186 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 20 Sep 2017 15:54:36 -0400 Subject: [PATCH 04/12] Upate test.json template --- .../run_tests/generated/tests.json.template | 1 + tools/run_tests/generated/tests.json | 201 ++++++++++++++++++ 2 files changed, 202 insertions(+) diff --git a/templates/tools/run_tests/generated/tests.json.template b/templates/tools/run_tests/generated/tests.json.template index 10ab2e445a4..9f82824a0ac 100644 --- a/templates/tools/run_tests/generated/tests.json.template +++ b/templates/tools/run_tests/generated/tests.json.template @@ -9,6 +9,7 @@ "platforms": tgt.platforms, "ci_platforms": tgt.ci_platforms, "gtest": tgt.gtest, + "benchmark": tgt.get("benchmark", False), "exclude_configs": tgt.get("exclude_configs", []), "exclude_iomgrs": tgt.get("exclude_iomgrs", []), "args": tgt.get("args", []), diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 72dfbc0b982..59c73ed8f43 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3,6 +3,7 @@ [ { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -25,6 +26,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -47,6 +49,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -69,6 +72,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -91,6 +95,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -113,6 +118,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -137,6 +143,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -159,6 +166,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -181,6 +189,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -203,6 +212,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -225,6 +235,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -247,6 +258,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -269,6 +281,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -291,6 +304,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -313,6 +327,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -335,6 +350,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -357,6 +373,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -379,6 +396,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -401,6 +419,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -423,6 +442,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -445,6 +465,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -469,6 +490,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -491,6 +513,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -515,6 +538,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -537,6 +561,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -559,6 +584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -583,6 +609,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -605,6 +632,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -623,6 +651,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -645,6 +674,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -665,6 +695,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -687,6 +718,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -709,6 +741,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -729,6 +762,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -749,6 +783,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -771,6 +806,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -793,6 +829,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -815,6 +852,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -837,6 +875,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -859,6 +898,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -881,6 +921,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -903,6 +944,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -925,6 +967,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -947,6 +990,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -969,6 +1013,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -991,6 +1036,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1013,6 +1059,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1035,6 +1082,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1057,6 +1105,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1079,6 +1128,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1101,6 +1151,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1123,6 +1174,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1145,6 +1197,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1167,6 +1220,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1189,6 +1243,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1211,6 +1266,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1233,6 +1289,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1255,6 +1312,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1277,6 +1335,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1299,6 +1358,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1323,6 +1383,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1345,6 +1406,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1367,6 +1429,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1387,6 +1450,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1409,6 +1473,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1431,6 +1496,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1449,6 +1515,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1467,6 +1534,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1489,6 +1557,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1511,6 +1580,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1533,6 +1603,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1555,6 +1626,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1575,6 +1647,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1591,6 +1664,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1613,6 +1687,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1635,6 +1710,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1657,6 +1733,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1679,6 +1756,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1701,6 +1779,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1723,6 +1802,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1745,6 +1825,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1765,6 +1846,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1787,6 +1869,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1809,6 +1892,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1831,6 +1915,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1853,6 +1938,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1875,6 +1961,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1897,6 +1984,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1921,6 +2009,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1943,6 +2032,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -1965,6 +2055,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux" ], @@ -1983,6 +2074,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2005,6 +2097,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2027,6 +2120,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2049,6 +2143,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2071,6 +2166,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2095,6 +2191,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2119,6 +2216,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2141,6 +2239,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2163,6 +2262,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2185,6 +2285,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2207,6 +2308,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2229,6 +2331,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2251,6 +2354,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2273,6 +2377,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2295,6 +2400,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2317,6 +2423,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2337,6 +2444,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2359,6 +2467,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2381,6 +2490,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2403,6 +2513,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2425,6 +2536,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2449,6 +2561,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2471,6 +2584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2493,6 +2607,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2517,6 +2632,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2539,6 +2655,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2561,6 +2678,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2585,6 +2703,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2609,6 +2728,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2631,6 +2751,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2653,6 +2774,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2675,6 +2797,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2695,6 +2818,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2717,6 +2841,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2739,6 +2864,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2761,6 +2887,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2783,6 +2910,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2805,6 +2933,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -2829,6 +2958,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2851,6 +2981,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2873,6 +3004,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2895,6 +3027,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2917,6 +3050,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2939,6 +3073,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2961,6 +3096,7 @@ "args": [ "--benchmark_min_time=4" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -2983,6 +3119,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3005,6 +3142,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3032,6 +3170,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3059,6 +3198,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3086,6 +3226,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3113,6 +3254,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3135,6 +3277,7 @@ "args": [ "--benchmark_min_time=0" ], + "benchmark": true, "ci_platforms": [ "linux", "mac", @@ -3155,6 +3298,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3177,6 +3321,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3199,6 +3344,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3221,6 +3367,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3241,6 +3388,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3267,6 +3415,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3289,6 +3438,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3311,6 +3461,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3333,6 +3484,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3355,6 +3507,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3377,6 +3530,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3399,6 +3553,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3421,6 +3576,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3443,6 +3599,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3465,6 +3622,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3487,6 +3645,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3511,6 +3670,7 @@ "args": [ "--generated_file_path=gens/src/proto/grpc/testing/" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3533,6 +3693,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3555,6 +3716,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3577,6 +3739,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3603,6 +3766,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3629,6 +3793,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3651,6 +3816,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3673,6 +3839,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3693,6 +3860,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3715,6 +3883,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3737,6 +3906,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3759,6 +3929,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3781,6 +3952,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3803,6 +3975,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3823,6 +3996,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3845,6 +4019,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3865,6 +4040,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3887,6 +4063,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3909,6 +4086,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3931,6 +4109,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3951,6 +4130,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3973,6 +4153,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -3995,6 +4176,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4017,6 +4199,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4039,6 +4222,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4059,6 +4243,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4081,6 +4266,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4104,6 +4290,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4124,6 +4311,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4146,6 +4334,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4170,6 +4359,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4194,6 +4384,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4218,6 +4409,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4242,6 +4434,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4266,6 +4459,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4290,6 +4484,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4314,6 +4509,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4338,6 +4534,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4362,6 +4559,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4386,6 +4584,7 @@ }, { "args": [], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4409,6 +4608,7 @@ "--test_bin_name=resolver_component_test_unsecure", "--running_under_bazel=false" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", @@ -4432,6 +4632,7 @@ "--test_bin_name=resolver_component_test", "--running_under_bazel=false" ], + "benchmark": false, "ci_platforms": [ "linux", "mac", From c9b945d23f7eca569d32a5905c41e9b97fd3ce90 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Thu, 12 Oct 2017 16:51:32 -0700 Subject: [PATCH 05/12] Turn on allow_flakes --- tools/internal_ci/linux/grpc_interop_matrix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal_ci/linux/grpc_interop_matrix.sh b/tools/internal_ci/linux/grpc_interop_matrix.sh index 6a9c38705ab..4c24c434883 100755 --- a/tools/internal_ci/linux/grpc_interop_matrix.sh +++ b/tools/internal_ci/linux/grpc_interop_matrix.sh @@ -22,4 +22,4 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/interop_matrix/run_interop_matrix_tests.py --language=all --release=all --report_file=sponge_log.xml --bq_result_table interop_results $@ +tools/interop_matrix/run_interop_matrix_tests.py --language=all --release=all --allow_flakes --report_file=sponge_log.xml --bq_result_table interop_results $@ From a584d992890a6d7b51bcff16876addcd49cfe3cf Mon Sep 17 00:00:00 2001 From: Ian Coolidge Date: Mon, 16 Oct 2017 10:28:13 -0700 Subject: [PATCH 06/12] cpu_linux: Don't spam sched_getcpu failures on qemu __NR_getcpu isn't implemented on qemu, and for some reason sysconf(_SC_NPROCESSORS_ONLN) returns the number of processers of the host system, giving a false indication that there is more than one cpu for the qemu case. Expand the init_num_cpus sequence to also run sched_getcpu once, and if that call isn't supported, initialize 'ncpus' to 1. Later, in gpr_cpu_current_cpu, use gpr_cpu_num_cores to avoid the system call in cases where we know it isn't supported, or if the ncpus is otherwise 1. --- src/core/lib/support/cpu_linux.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/lib/support/cpu_linux.cc b/src/core/lib/support/cpu_linux.cc index 22806684421..53619caa5f1 100644 --- a/src/core/lib/support/cpu_linux.cc +++ b/src/core/lib/support/cpu_linux.cc @@ -38,8 +38,9 @@ static int ncpus = 0; static void init_num_cpus() { /* This must be signed. sysconf returns -1 when the number cannot be determined */ + int cpu = sched_getcpu(); ncpus = (int)sysconf(_SC_NPROCESSORS_ONLN); - if (ncpus < 1) { + if (ncpus < 1 || cpu < 0) { gpr_log(GPR_ERROR, "Cannot determine number of CPUs: assuming 1"); ncpus = 1; } @@ -56,6 +57,9 @@ unsigned gpr_cpu_current_cpu(void) { // sched_getcpu() is undefined on musl return 0; #else + if (gpr_cpu_num_cores() == 1) { + return 0; + } int cpu = sched_getcpu(); if (cpu < 0) { gpr_log(GPR_ERROR, "Error determining current CPU: %s\n", strerror(errno)); From 58b2d85a21b60a6961ae83459a70e664080e4a54 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 16 Oct 2017 14:30:02 -0700 Subject: [PATCH 07/12] Checking in tools --- tools/debug/core/error_ref_leak.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tools/debug/core/error_ref_leak.py diff --git a/tools/debug/core/error_ref_leak.py b/tools/debug/core/error_ref_leak.py new file mode 100644 index 00000000000..99f986a97bc --- /dev/null +++ b/tools/debug/core/error_ref_leak.py @@ -0,0 +1,27 @@ +#!/bin/python + +import sys +import re + +data = sys.stdin.readlines() + +errs = [] +for line in data: + if re.search(r'error.cc', line): + line = line.partition('error.cc:')[-1] + line = re.sub(r'\d+] ', r'', line) + line = line.strip().split() + err = line[0].strip(":") + if line[1] == "create": + assert(err not in errs) + errs.append(err) + elif line[0] == "realloc": + errs.remove(line[1]) + errs.append(line[3]) + elif line[1] == "1" and line[3] == "0": + # print line + # print err, errs + assert(err in errs) + errs.remove(err) + +print "leaked:", errs From 3ea4e5158a8d24a2bf9437d836ed662b4afd3772 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 16 Oct 2017 14:45:24 -0700 Subject: [PATCH 08/12] Add copyright and usage --- tools/debug/core/error_ref_leak.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tools/debug/core/error_ref_leak.py b/tools/debug/core/error_ref_leak.py index 99f986a97bc..6582328a5b8 100644 --- a/tools/debug/core/error_ref_leak.py +++ b/tools/debug/core/error_ref_leak.py @@ -1,4 +1,23 @@ -#!/bin/python +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Reads stdin to find error_refcount log lines, and prints reference leaks +# to stdout + +# usege: python error_ref_leak < logfile.txt import sys import re @@ -7,7 +26,9 @@ data = sys.stdin.readlines() errs = [] for line in data: + # if we care about the line if re.search(r'error.cc', line): + # str manip to cut off left part of log line line = line.partition('error.cc:')[-1] line = re.sub(r'\d+] ', r'', line) line = line.strip().split() @@ -18,9 +39,8 @@ for line in data: elif line[0] == "realloc": errs.remove(line[1]) errs.append(line[3]) + # explicitly look for the last dereference elif line[1] == "1" and line[3] == "0": - # print line - # print err, errs assert(err in errs) errs.remove(err) From 77013000e7fb7740a6e6f7cf76eb47f8a282baf2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 16 Oct 2017 16:11:37 -0700 Subject: [PATCH 09/12] Working benchmark parallelization --- tools/run_tests/run_tests.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 340e848e7f8..588784353a9 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -149,10 +149,8 @@ class Config(object): for k, v in environ.items(): actual_environ[k] = v if not flaky and shortname and shortname in flaky_tests: - print('Setting %s to flaky' % shortname) flaky = True if shortname in shortname_to_cpu: - print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, shortname_to_cpu[shortname])) cpu_cost = shortname_to_cpu[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, @@ -339,19 +337,24 @@ class CLanguage(object): # and filter test runs. We use them to split each individual test # into its own JobSpec, and thus into its own process. if 'benchmark' in target and target['benchmark']: - list_test_command = '--benchmark_list_tests' - filter_test_command = '--benchmark_filter=%s' + with open(os.devnull, 'w') as fnull: + tests = subprocess.check_output([binary, '--benchmark_list_tests'], + stderr=fnull) + base = None + for line in tests.split('\n'): + test = line.strip() + cmdline = [binary, '--benchmark_filter=%s$' % test] + target['args'] + out.append(self.config.job_spec(cmdline, + shortname='%s:%s %s' % (binary, test, shortname_ext), + cpu_cost=cpu_cost, + timeout_seconds=_DEFAULT_TIMEOUT_SECONDS * timeout_scaling, + environ=env)) elif 'gtest' in target and target['gtest']: - list_test_command = '--gtest_list_tests' - filter_test_command = '--gtest_filter=%s' - - if list_test_command: - # here we parse the output of --gtest_list_tests (or - # --benchmark_list_tests)to build up a complete list of - # the tests contained in a binary for each test, we then + # here we parse the output of --gtest_list_tests to build up a complete + # list of the tests contained in a binary for each test, we then # add a job to run, filtering for just that test. with open(os.devnull, 'w') as fnull: - tests = subprocess.check_output([binary, list_test_command], + tests = subprocess.check_output([binary, '--gtest_list_tests'], stderr=fnull) base = None for line in tests.split('\n'): @@ -364,7 +367,7 @@ class CLanguage(object): assert base is not None assert line[1] == ' ' test = base + line.strip() - cmdline = [binary, filter_test_command % test] + target['args'] + cmdline = [binary, '--gtest_filter=%s' % test] + target['args'] out.append(self.config.job_spec(cmdline, shortname='%s %s' % (' '.join(cmdline), shortname_ext), cpu_cost=cpu_cost, From a33eb8d7004c432880ffaaa11b591e59348df68d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 16 Oct 2017 19:18:50 -0700 Subject: [PATCH 10/12] Fix breakage --- src/core/lib/security/credentials/oauth2/oauth2_credentials.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc index f52a424e363..7867105f567 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc @@ -262,7 +262,7 @@ static bool oauth2_token_fetcher_get_request_metadata( grpc_mdelem cached_access_token_md = GRPC_MDNULL; gpr_mu_lock(&c->mu); if (!GRPC_MDISNULL(c->access_token_md) && - (c->token_expiration + grpc_exec_ctx_now(exec_ctx) > refresh_threshold)) { + (c->token_expiration - grpc_exec_ctx_now(exec_ctx) > refresh_threshold)) { cached_access_token_md = GRPC_MDELEM_REF(c->access_token_md); } if (!GRPC_MDISNULL(cached_access_token_md)) { From 1f42fae4c5125c33f7b953d0d0073af8f8e4397e Mon Sep 17 00:00:00 2001 From: Krasimir Georgiev Date: Tue, 17 Oct 2017 08:38:20 +0200 Subject: [PATCH 11/12] Fix initialization bug in channel creation --- src/cpp/client/create_channel.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/client/create_channel.cc b/src/cpp/client/create_channel.cc index e2893c8f3ca..142d7beabe5 100644 --- a/src/cpp/client/create_channel.cc +++ b/src/cpp/client/create_channel.cc @@ -38,7 +38,7 @@ std::shared_ptr CreateCustomChannel( const grpc::string& target, const std::shared_ptr& creds, const ChannelArguments& args) { - internal::GrpcLibrary + GrpcLibraryCodegen init_lib; // We need to call init in case of a bad creds. return creds ? creds->CreateChannel(target, args) From 53cec0fc8fa170aefc641aaade014d8642f33302 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 17 Oct 2017 06:46:19 -0700 Subject: [PATCH 12/12] Update create_channel.cc --- src/cpp/client/create_channel.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cpp/client/create_channel.cc b/src/cpp/client/create_channel.cc index 142d7beabe5..de67281dd45 100644 --- a/src/cpp/client/create_channel.cc +++ b/src/cpp/client/create_channel.cc @@ -38,8 +38,7 @@ std::shared_ptr CreateCustomChannel( const grpc::string& target, const std::shared_ptr& creds, const ChannelArguments& args) { - GrpcLibraryCodegen - init_lib; // We need to call init in case of a bad creds. + GrpcLibraryCodegen init_lib; // We need to call init in case of a bad creds. return creds ? creds->CreateChannel(target, args) : CreateChannelInternal("", grpc_lame_client_channel_create(