From fc661c35a20a7cc85f76e5e0c502700f42e27cf2 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <pbonzini@redhat.com>
Date: Sat, 15 Jan 2022 23:36:42 +0100
Subject: [PATCH] interpreter: support for forcibly verbose logging of some
 tests

Add a new keyword argument to test() and benchmark(), completing the
implementation of the feature.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 docs/markdown/snippets/test-verbose.md       | 6 ++++++
 docs/yaml/functions/benchmark.yaml           | 8 ++++++++
 docs/yaml/functions/test.yaml                | 7 +++++++
 mesonbuild/backend/backends.py               | 2 +-
 mesonbuild/interpreter/interpreter.py        | 4 +++-
 mesonbuild/interpreter/interpreterobjects.py | 3 ++-
 6 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 docs/markdown/snippets/test-verbose.md

diff --git a/docs/markdown/snippets/test-verbose.md b/docs/markdown/snippets/test-verbose.md
new file mode 100644
index 000000000..6d9aa0f98
--- /dev/null
+++ b/docs/markdown/snippets/test-verbose.md
@@ -0,0 +1,6 @@
+## New keyword argument `verbose` for tests and benchmarks
+
+The new keyword argument `verbose` can be used to mark tests and benchmarks
+that must always be logged verbosely on the console.  This is particularly
+useful for long-running tests, or when a single Meson test() is wrapping
+an external test harness.
diff --git a/docs/yaml/functions/benchmark.yaml b/docs/yaml/functions/benchmark.yaml
index da465aa83..3082fbe12 100644
--- a/docs/yaml/functions/benchmark.yaml
+++ b/docs/yaml/functions/benchmark.yaml
@@ -104,3 +104,11 @@ kwargs:
       The starting order of tests with identical priorities is
       implementation-defined. The default priority is 0, negative numbers are
       permitted.
+
+  verbose:
+    type: bool
+    since: 0.62.0
+    default: false
+    description: |
+      if true, forces the test results to be logged as if `--verbose` was passed
+      to `meson test`.
diff --git a/docs/yaml/functions/test.yaml b/docs/yaml/functions/test.yaml
index 96a2b286d..bc9ad03eb 100644
--- a/docs/yaml/functions/test.yaml
+++ b/docs/yaml/functions/test.yaml
@@ -33,6 +33,13 @@ description: |
   test(..., env: nomalloc, ...)
   ```
 
+  In addition to running individual executables as test cases, `test()`
+  can also be used to invoke an external test harness.  In this case,
+  it is best to use `verbose: true` *(since 0.62.0)* and, if supported
+  by the external harness, `protocol: 'tap'` *(since 0.50.0)*.  This will
+  ensure that Meson logs each subtest as it runs, instead of including
+  the whole log at the end of the run.
+
   Defined tests can be run in a backend-agnostic way by calling
   `meson test` inside the build dir, or by using backend-specific
   commands, such as `ninja test` or `msbuild RUN_TESTS.vcxproj`.
diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py
index 8729c8e35..bc4a02cf6 100644
--- a/mesonbuild/backend/backends.py
+++ b/mesonbuild/backend/backends.py
@@ -1150,7 +1150,7 @@ class Backend:
                                    isinstance(exe, build.Executable),
                                    [x.get_id() for x in depends],
                                    self.environment.coredata.version,
-                                   False)
+                                   t.verbose)
             arr.append(ts)
         return arr
 
diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py
index 768183494..c83d09eed 100644
--- a/mesonbuild/interpreter/interpreter.py
+++ b/mesonbuild/interpreter/interpreter.py
@@ -215,6 +215,7 @@ TEST_KWARGS: T.List[KwargInfo] = [
     ENV_KW,
     DEPENDS_KW.evolve(since='0.46.0'),
     KwargInfo('suite', ContainerTypeInfo(list, str), listify=True, default=['']),  # yes, a list of empty string
+    KwargInfo('verbose', bool, default=False, since='0.62.0'),
 ]
 
 permitted_dependency_kwargs = {
@@ -1972,7 +1973,8 @@ external dependencies (including libraries) must go to "dependencies".''')
                     kwargs['timeout'],
                     kwargs['workdir'],
                     kwargs['protocol'],
-                    kwargs['priority'])
+                    kwargs['priority'],
+                    kwargs['verbose'])
 
     def add_test(self, node: mparser.BaseNode, args: T.List, kwargs: T.Dict[str, T.Any], is_base_test: bool):
         t = self.make_test(node, args, kwargs)
diff --git a/mesonbuild/interpreter/interpreterobjects.py b/mesonbuild/interpreter/interpreterobjects.py
index 2656f1405..9c2481c06 100644
--- a/mesonbuild/interpreter/interpreterobjects.py
+++ b/mesonbuild/interpreter/interpreterobjects.py
@@ -636,7 +636,7 @@ class Test(MesonInterpreterObject):
                  cmd_args: T.List[T.Union[str, mesonlib.File, build.Target]],
                  env: build.EnvironmentVariables,
                  should_fail: bool, timeout: int, workdir: T.Optional[str], protocol: str,
-                 priority: int):
+                 priority: int, verbose: bool):
         super().__init__()
         self.name = name
         self.suite = listify(suite)
@@ -651,6 +651,7 @@ class Test(MesonInterpreterObject):
         self.workdir = workdir
         self.protocol = TestProtocol.from_str(protocol)
         self.priority = priority
+        self.verbose = verbose
 
     def get_exe(self) -> T.Union[ExternalProgram, build.Executable, build.CustomTarget]:
         return self.exe