project tests: log the reason why a test is skipped

We expose a reason after the string 'MESON_SKIP_TEST', but it is
actually ignored when running the test, so it is only useful as
documentation and really might as well be a comment.

Make it even more useful by actually printing that string after the
'[SKIPPED]' message.

Also, sometimes a test can be skipped for multiple reasons, and it would
be useful to know which one occurred.
pull/10152/head
Eli Schwartz 3 years ago
parent ce7a67e511
commit 105bbaabdd
No known key found for this signature in database
GPG Key ID: CEB167EFB5722BD6
  1. 18
      run_project_tests.py
  2. 20
      run_single_test.py

@ -1173,6 +1173,12 @@ class LogRunFuture:
RunFutureUnion = T.Union[TestRunFuture, LogRunFuture]
def test_emits_skip_msg(line: str) -> bool:
for prefix in {'Problem encountered', 'Assert failed', 'Failed to configure the CMake subproject'}:
if f'{prefix}: MESON_SKIP_TEST' in line:
return True
return False
def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
log_name_base: str,
failfast: bool,
@ -1281,10 +1287,19 @@ def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
if result is None:
# skipped due to skipped category skip or 'tools:' or 'skip_on_env:'
is_skipped = True
skip_reason = 'not run because preconditions were not met'
skip_as_expected = True
else:
# skipped due to test outputting 'MESON_SKIP_TEST'
is_skipped = 'MESON_SKIP_TEST' in result.stdo
for l in result.stdo.splitlines():
if test_emits_skip_msg(l):
is_skipped = True
offset = l.index('MESON_SKIP_TEST') + 16
skip_reason = l[offset:].strip()
break
else:
is_skipped = False
skip_reason = ''
if not skip_dont_care(t):
skip_as_expected = (is_skipped == t.skip_expected)
else:
@ -1295,6 +1310,7 @@ def _run_tests(all_tests: T.List[T.Tuple[str, T.List[TestDef], bool]],
if is_skipped and skip_as_expected:
f.update_log(TestStatus.SKIP)
safe_print(bold('Reason:'), skip_reason)
current_test = ET.SubElement(current_suite, 'testcase', {'name': testname, 'classname': t.category})
ET.SubElement(current_test, 'skipped', {})
continue

@ -13,7 +13,7 @@ import pathlib
import typing as T
from mesonbuild import mlog
from run_project_tests import TestDef, load_test_json, run_test, BuildStep
from run_project_tests import TestDef, load_test_json, run_test, BuildStep, test_emits_skip_msg
from run_project_tests import setup_commands, detect_system_compiler, print_tool_versions
if T.TYPE_CHECKING:
@ -59,7 +59,21 @@ def main() -> None:
results = [run_test(t, t.args, should_fail(t.path), args.use_tmpdir) for t in tests]
failed = False
for test, result in zip(tests, results):
if (result is None) or ('MESON_SKIP_TEST' in result.stdo):
if result is None:
is_skipped = True
skip_reason = 'not run because preconditions were not met'
else:
for l in result.stdo.splitlines():
if test_emits_skip_msg(l):
is_skipped = True
offset = l.index('MESON_SKIP_TEST') + 16
skip_reason = l[offset:].strip()
break
else:
is_skipped = False
skip_reason = ''
if is_skipped:
msg = mlog.yellow('SKIP:')
elif result.msg:
msg = mlog.red('FAIL:')
@ -67,6 +81,8 @@ def main() -> None:
else:
msg = mlog.green('PASS:')
mlog.log(msg, *test.display_name())
if skip_reason:
mlog.log(mlog.bold('Reason:'), skip_reason)
if result is not None and result.msg and 'MESON_SKIP_TEST' not in result.stdo:
mlog.log('reason:', result.msg)
if result.step is BuildStep.configure:

Loading…
Cancel
Save