add a script to run a single test

As a Meson developer it's often frustrating to have a single functional
test with a regression. These tests can be awkward to reproduce,
especially when they make use of a test.json file. This script provides
a simmple interface to call functional tests 1 at a time, regardless of
whether they use a test.json or not. If they do use a test.json, and
have a matrix, then the `--subtest` option can be used to select spcific
combinations, for example:
```sh
./run_single_test.py "test cases/frameworks/15 llvm" --subtest 1
```
will run only the second (zero indexed of course) subtest from the llvm
test cases.

This is not a super elegent script, but this is super useful.

run_mypy.py

@@ -39,6 +39,7 @@ modules = [
     'mesonbuild/optinterpreter.py',
 
     'run_mypy.py',
+    'run_single_test.py',
     'tools'
 ]
 

run_single_test.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# SPDX-license-identifier: Apache-2.0
+# Copyright © 2021 Intel Corporation
+
+"""Script for running a single project test.
+
+This script is meant for Meson developers who want to run a single project
+test, with all of the rules from the test.json file loaded.
+"""
+
+import argparse
+import pathlib
+import shutil
+import typing as T
+
+from mesonbuild import environment
+from mesonbuild import mlog
+from mesonbuild import mesonlib
+from run_project_tests import TestDef, load_test_json, run_test, BuildStep
+from run_tests import get_backend_commands, guess_backend, get_fake_options
+
+if T.TYPE_CHECKING:
+    try:
+        from typing import Protocol
+    except ImportError:
+        # Mypy gets grump about this even though it's fine
+        from typing_extensions import Protocol  # type: ignore
+
+    class ArgumentType(Protocol):
+
+        """Typing information for command line arguments."""
+
+        case: pathlib.Path
+        subtests: T.List[int]
+        backend: str
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('case', type=pathlib.Path, help='The test case to run')
+    parser.add_argument('--subtest', type=int, action='append', dest='subtests', help='which subtests to run')
+    parser.add_argument('--backend', action='store', help="Which backend to use")
+    args = T.cast('ArgumentType', parser.parse_args())
+
+    test = TestDef(args.case, args.case.stem, [])
+    tests = load_test_json(test, False)
+    if args.subtests:
+        tests = [t for i, t in enumerate(tests) if i in args.subtests]
+
+    with mesonlib.TemporaryDirectoryWinProof() as build_dir:
+        fake_opts = get_fake_options('/')
+        env = environment.Environment(None, build_dir, fake_opts)
+        try:
+            comp = env.compiler_from_language('c', mesonlib.MachineChoice.HOST).get_id()
+        except mesonlib.MesonException:
+            raise RuntimeError('Could not detect C compiler')
+
+    backend, backend_args = guess_backend(args.backend, shutil.which('msbuild'))
+    _cmds = get_backend_commands(backend, False)
+    commands = (_cmds[0], _cmds[1], _cmds[3], _cmds[4])
+
+    results = [run_test(t, t.args, comp, backend, backend_args, commands, False, True) for t in tests]
+    failed = False
+    for test, result in zip(tests, results):
+        if result is None:
+            msg = mlog.yellow('SKIP:')
+        elif result.msg:
+            msg = mlog.red('FAIL:')
+            failed = True
+        else:
+            msg = mlog.green('PASS:')
+        mlog.log(msg, test.display_name())
+        if result.msg:
+            mlog.log('reason:', result.msg)
+            if result.step is BuildStep.configure:
+                # For configure failures, instead of printing stdout,
+                # print the meson log if available since it's a superset
+                # of stdout and often has very useful information.
+                mlog.log(result.mlog)
+            else:
+                mlog.log(result.stdo)
+            for cmd_res in result.cicmds:
+                mlog.log(cmd_res)
+            mlog.log(result.stde)
+
+    exit(1 if failed else 0)
+
+if __name__ == "__main__":
+    main()

run_tests.py

@@ -118,7 +118,8 @@ class FakeCompilerOptions:
     def __init__(self):
         self.value = []
 
-def get_fake_options(prefix=''):
+# TODO: use a typing.Protocol here
+def get_fake_options(prefix: str = '') -> argparse.Namespace:
     opts = argparse.Namespace()
     opts.native_file = []
     opts.cross_file = None