diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 33c592cdf..407507a41 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2080,14 +2080,14 @@ requirements use the version keyword argument instead.''') workdir = None if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') - suite = mesonlib.stringlistify(kwargs.get('suite', '')) - if self.is_subproject(): - newsuite = [] - for s in suite: - if len(s) > 0: - s = '.' + s - newsuite.append(self.subproject.replace(' ', '_').replace('.', '_') + s) - suite = newsuite + suite = [] + for s in mesonlib.stringlistify(kwargs.get('suite', '')): + if len(s) > 0: + s = ':' + s + if self.is_subproject(): + suite.append(self.subproject.replace(' ', '_').replace(':', '_') + s) + else: + suite.append(self.build.project_name.replace(' ', '_').replace(':', '_') + s) t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, timeout, workdir) if is_base_test: self.build.tests.append(t) diff --git a/mesontest.py b/mesontest.py index df71d0419..9c7559d6b 100755 --- a/mesontest.py +++ b/mesontest.py @@ -64,8 +64,10 @@ parser.add_argument('--wrapper', default=None, dest='wrapper', help='wrapper to run tests with (e.g. Valgrind)') parser.add_argument('-C', default='.', dest='wd', help='directory to cd into before running') -parser.add_argument('--suite', default=None, dest='suite', +parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', help='Only run tests belonging to the given suite.') +parser.add_argument('--no-suite', default=[], dest='exclude_suites', action='append', metavar='SUITE', + help='Do not run tests belonging to the given suite.') parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', help='Do not split stderr and stdout in test logs.') parser.add_argument('--print-errorlogs', default=False, action='store_true', @@ -155,10 +157,12 @@ class TestHarness: self.timeout_count = 0 self.is_run = False self.cant_rebuild = False + self.tests = None + self.suites = None if self.options.benchmark: - self.datafile = os.path.join(options.wd, 'meson-private/meson_benchmark_setup.dat') + self.load_datafile(os.path.join(options.wd, 'meson-private/meson_benchmark_setup.dat')) else: - self.datafile = os.path.join(options.wd, 'meson-private/meson_test_setup.dat') + self.load_datafile(os.path.join(options.wd, 'meson-private/meson_test_setup.dat')) def rebuild_all(self): if not os.path.isfile(os.path.join(self.options.wd, 'build.ninja')): @@ -325,16 +329,56 @@ class TestHarness: self.run_tests(tests) return self.fail_count - def get_tests(self): + def split_suite_string(suite): + if ':' in suite: + return suite.split(':', 1) + else: + return (suite, "") + + def test_in_suites(test, suites): + for suite in suites: + (prj_match, st_match) = TestHarness.split_suite_string(suite) + for prjst in test.suite: + (prj, st) = TestHarness.split_suite_string(prjst) + if prj_match and prj != prj_match: + continue + if st_match and st != st_match: + continue + return True + return False + + def test_suitable(self, test): + return (len(self.options.include_suites) == 0 or TestHarness.test_in_suites(test, self.options.include_suites)) \ + and not TestHarness.test_in_suites(test, self.options.exclude_suites) + + def load_suites(self): + ss = set() + for t in self.tests: + for s in t.suite: + ss.add(s) + self.suites = list(ss) + + def load_tests(self): with open(self.datafile, 'rb') as f: - tests = pickle.load(f) + self.tests = pickle.load(f) - if not tests: + def load_datafile(self, datafile): + self.datafile = datafile + self.load_tests() + self.load_suites() + + def get_tests(self): + if not self.tests: print('No tests defined.') return [] - if self.options.suite: - tests = [t for t in tests if self.options.suite in t.suite] + if len(self.options.include_suites) or len(self.options.exclude_suites): + tests = [] + for tst in self.tests: + if self.test_suitable(tst): + tests.append(tst) + else: + tests = self.tests if self.options.args: tests = [t for t in tests if t.name in self.options.args] @@ -384,8 +428,15 @@ class TestHarness: assert(isinstance(wrap, list)) return wrap - def get_suites(self, tests): - return set([test.suite[0] for test in tests]) + def get_pretty_suite(self, test, tests): + if len(self.suites) > 1: + rv = TestHarness.split_suite_string(test.suite[0])[0] + s = "+".join(TestHarness.split_suite_string(s)[1] for s in test.suite) + if len(s): + rv += ":" + return rv + s + " / " + test.name + else: + return test.name def run_tests(self, tests): try: @@ -399,13 +450,7 @@ class TestHarness: for i in range(self.options.repeat): for i, test in enumerate(tests): - if test.suite[0] == '': - visible_name = test.name - else: - if self.options.suite is not None: - visible_name = self.options.suite + ' / ' + test.name - else: - visible_name = test.suite[0] + ' / ' + test.name + visible_name = self.get_pretty_suite(test, tests) if self.options.gdb: test.timeout = None @@ -461,13 +506,8 @@ class TestHarness: def list_tests(th): tests = th.get_tests() - print_suites = True if len(th.get_suites(tests)) != 1 else False - for i in tests: - if print_suites: - print("%s / %s" % (i.suite[0], i.name)) - else: - print("%s" % i.name) - + for t in tests: + print(th.get_pretty_suite(t, tests)) def merge_suite_options(options): buildfile = os.path.join(options.wd, 'meson-private/build.dat') diff --git a/run_unittests.py b/run_unittests.py index d1c192f04..adc0e7fc9 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -418,6 +418,61 @@ class LinuxlikeTests(unittest.TestCase): self.assertTrue('TEST_ENV is set' in vg_log) self.assertTrue('Memcheck' in vg_log) + def assertFailedTestCount(self, failure_count, command): + try: + self._run(command) + self.assertEqual(0, failure_count, 'Expected %d tests to fail.' % failure_count) + except subprocess.CalledProcessError as e: + self.assertEqual(e.returncode, failure_count) + + def test_suite_selection(self): + testdir = os.path.join(self.unit_test_dir, '4 suite selection') + self.init(testdir) + self.build() + + self.assertFailedTestCount(3, self.mtest_command) + + self.assertFailedTestCount(0, self.mtest_command + ['--suite', ':success']) + self.assertFailedTestCount(3, self.mtest_command + ['--suite', ':fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', ':success']) + self.assertFailedTestCount(0, self.mtest_command + ['--no-suite', ':fail']) + + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'mainprj']) + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc']) + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail']) + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjmix']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'mainprj']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjfail']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjmix']) + + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'mainprj:fail']) + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'mainprj:success']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'mainprj:fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'mainprj:success']) + + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail:fail']) + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjfail:success']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjfail:fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjfail:success']) + + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc:fail']) + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc:success']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc:fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc:success']) + + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjmix:fail']) + self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjmix:success']) + self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjmix:fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjmix:success']) + + self.assertFailedTestCount(2, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix:fail']) + self.assertFailedTestCount(3, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj']) + self.assertFailedTestCount(2, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj', '--no-suite', 'subprjmix:fail']) + self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj', '--no-suite', 'subprjmix:fail', 'mainprj-failing_test']) + + self.assertFailedTestCount(1, self.mtest_command + ['--no-suite', 'subprjfail:fail', '--no-suite', 'subprjmix:fail']) + def _test_stds_impl(self, testdir, compiler, p): lang_std = p + '_std' # Check that all the listed -std=xxx options for this compiler work diff --git a/test cases/unit/4 suite selection/failing_test.c b/test cases/unit/4 suite selection/failing_test.c new file mode 100644 index 000000000..bf0090735 --- /dev/null +++ b/test cases/unit/4 suite selection/failing_test.c @@ -0,0 +1 @@ +int main() { return -1 ; } diff --git a/test cases/unit/4 suite selection/meson.build b/test cases/unit/4 suite selection/meson.build new file mode 100644 index 000000000..d3d4e1aba --- /dev/null +++ b/test cases/unit/4 suite selection/meson.build @@ -0,0 +1,13 @@ +project('mainprj', 'c') + +subproject('subprjfail') +subproject('subprjsucc') +subproject('subprjmix') + +test('mainprj-failing_test', + executable('failing_test', 'failing_test.c'), + suite : 'fail') + +test('mainprj-successful_test', + executable('successful_test', 'successful_test.c'), + suite : 'success') diff --git a/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c b/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c new file mode 100644 index 000000000..bf0090735 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c @@ -0,0 +1 @@ +int main() { return -1 ; } diff --git a/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build b/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build new file mode 100644 index 000000000..d95f271b7 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build @@ -0,0 +1,5 @@ +project('subprjfail', 'c') + +test('subprjfail-failing_test', + executable('failing_test', 'failing_test.c'), + suite : 'fail') diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c b/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c new file mode 100644 index 000000000..bf0090735 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c @@ -0,0 +1 @@ +int main() { return -1 ; } diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build b/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build new file mode 100644 index 000000000..1d0eeff2e --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build @@ -0,0 +1,9 @@ +project('subprjmix', 'c') + +test('subprjmix-failing_test', + executable('failing_test', 'failing_test.c'), + suite : 'fail') + +test('subprjmix-successful_test', + executable('successful_test', 'successful_test.c'), + suite : 'success') diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c b/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c new file mode 100644 index 000000000..29c0525a5 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c @@ -0,0 +1 @@ +int main() { return 0 ; } diff --git a/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build b/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build new file mode 100644 index 000000000..8dafd65b0 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build @@ -0,0 +1,5 @@ +project('subprjsucc', 'c') + +test('subprjsucc-successful_test', + executable('successful_test', 'successful_test.c'), + suite : 'success') diff --git a/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c b/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c new file mode 100644 index 000000000..29c0525a5 --- /dev/null +++ b/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c @@ -0,0 +1 @@ +int main() { return 0 ; } diff --git a/test cases/unit/4 suite selection/successful_test.c b/test cases/unit/4 suite selection/successful_test.c new file mode 100644 index 000000000..29c0525a5 --- /dev/null +++ b/test cases/unit/4 suite selection/successful_test.c @@ -0,0 +1 @@ +int main() { return 0 ; }