interpreter, mtest: introduce add_test_setup(exclude_suites: ...)

This new keyword argument makes it possible to run specific
test setups only on a subset of the tests.  For example, to
mark some tests as slow and avoid running them by default:

    add_test_setup('quick', exclude_suites: ['slow'], is_default: true)
    add_test_setup('slow')

It will then be possible to run the slow tests with either
`meson test --setup slow` or `meson test --suite slow`.

docs/markdown/Reference-manual.md

@@ -125,6 +125,9 @@ the following:
 - `is_default` *(since 0.49.0)*: a bool to set whether this is the default test setup.
   If `true`, the setup will be used whenever `meson test` is run
   without the `--setup` option.
+- `exclude_suites` *(since 0.57.0)*: a list of test suites that should be
+  excluded when using this setup.  Suites specified in the `--suite` option
+  to `meson test` will always run, overriding `add_test_setup` if necessary.
 
 To use the test setup, run `meson test --setup=*name*` inside the
 build dir.

mesonbuild/build.py

@@ -2617,11 +2617,13 @@ class Data:
 
 class TestSetup:
     def __init__(self, exe_wrapper: T.Optional[T.List[str]], gdb: bool,
-                 timeout_multiplier: int, env: EnvironmentVariables):
+                 timeout_multiplier: int, env: EnvironmentVariables,
+                 exclude_suites: T.List[str]):
         self.exe_wrapper = exe_wrapper
         self.gdb = gdb
         self.timeout_multiplier = timeout_multiplier
         self.env = env
+        self.exclude_suites = exclude_suites
 
 def get_sources_string_names(sources, backend):
     '''

mesonbuild/interpreter.py

@@ -2300,7 +2300,8 @@ permitted_kwargs = {'add_global_arguments': {'language', 'native'},
                     'add_languages': {'required', 'native'},
                     'add_project_link_arguments': {'language', 'native'},
                     'add_project_arguments': {'language', 'native'},
-                    'add_test_setup': {'exe_wrapper', 'gdb', 'timeout_multiplier', 'env', 'is_default'},
+                    'add_test_setup': {'exe_wrapper', 'gdb', 'timeout_multiplier', 'env', 'is_default',
+                                       'exclude_suites'},
                     'benchmark': _base_test_args,
                     'build_target': known_build_target_kwargs,
                     'configure_file': {'input',
@@ -4687,8 +4688,10 @@ different subdirectory.
                 raise InterpreterException('\'%s\' is already set as default. '
                                            'is_default can be set to true only once' % self.build.test_setup_default_name)
             self.build.test_setup_default_name = setup_name
+        exclude_suites = mesonlib.stringlistify(kwargs.get('exclude_suites', []))
         env = self.unpack_env_kwarg(kwargs)
-        self.build.test_setups[setup_name] = build.TestSetup(exe_wrapper, gdb, timeout_multiplier, env)
+        self.build.test_setups[setup_name] = build.TestSetup(exe_wrapper, gdb, timeout_multiplier, env,
+                                                             exclude_suites)
 
     @permittedKwargs(permitted_kwargs['add_global_arguments'])
     @stringArgs

mesonbuild/mtest.py

@@ -1654,9 +1654,20 @@ class TestHarness:
         return False
 
     def test_suitable(self, test: TestSerialisation) -> bool:
-        return ((not self.options.include_suites or
-                TestHarness.test_in_suites(test, self.options.include_suites)) and not
-                TestHarness.test_in_suites(test, self.options.exclude_suites))
+        if TestHarness.test_in_suites(test, self.options.exclude_suites):
+            return False
+
+        if self.options.include_suites:
+            # Both force inclusion (overriding add_test_setup) and exclude
+            # everything else
+            return TestHarness.test_in_suites(test, self.options.include_suites)
+
+        if self.options.setup:
+            setup = self.get_test_setup(test)
+            if TestHarness.test_in_suites(test, setup.exclude_suites):
+                return False
+
+        return True
 
     def tests_from_args(self, tests: T.List[TestSerialisation]) -> T.Generator[TestSerialisation, None, None]:
         '''
@@ -1685,14 +1696,7 @@ class TestHarness:
             print('No tests defined.')
             return []
 
-        if self.options.include_suites or self.options.exclude_suites:
-            tests = []
-            for tst in self.tests:
-                if self.test_suitable(tst):
-                    tests.append(tst)
-        else:
-            tests = self.tests
-
+        tests = [t for t in self.tests if self.test_suitable(t)]
         if self.options.args:
             tests = list(self.tests_from_args(tests))
 

run_unittests.py

@@ -2351,6 +2351,16 @@ class AllPlatformTests(BasePlatformTests):
         self._run(self.mtest_command + ['--setup=onlyenv3'])
         # Setup with only a timeout works
         self._run(self.mtest_command + ['--setup=timeout'])
+        # Setup that skips test works
+        self._run(self.mtest_command + ['--setup=good'])
+        with open(os.path.join(self.logdir, 'testlog-good.txt')) as f:
+            exclude_suites_log = f.read()
+        self.assertFalse('buggy' in exclude_suites_log)
+        # --suite overrides add_test_setup(xclude_suites)
+        self._run(self.mtest_command + ['--setup=good', '--suite', 'buggy'])
+        with open(os.path.join(self.logdir, 'testlog-good.txt')) as f:
+            include_suites_log = f.read()
+        self.assertTrue('buggy' in include_suites_log)
 
     def test_testsetup_selection(self):
         testdir = os.path.join(self.unit_test_dir, '14 testsetup selection')

test cases/unit/2 testsetups/meson.build

@@ -12,7 +12,7 @@ add_test_setup('valgrind',
   env : env)
 
 buggy = executable('buggy', 'buggy.c', 'impl.c')
-test('Test buggy', buggy)
+test('Test buggy', buggy, suite: ['buggy'])
 
 envcheck = find_program('envcheck.py')
 test('test-env', envcheck)
@@ -23,3 +23,4 @@ add_test_setup('onlyenv2', env : 'TEST_ENV=1')
 add_test_setup('onlyenv3', env : ['TEST_ENV=1'])
 add_test_setup('wrapper', exe_wrapper : [vg, '--error-exitcode=1'])
 add_test_setup('timeout', timeout_multiplier : 20)
+add_test_setup('good', exclude_suites : 'buggy')