From 542255993cce730b01d8bf79bf48aa8f5ad36fc9 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:29:16 -0700 Subject: [PATCH 1/9] mtest: Replace if (bool) { return bool; } with return bool; --- mesonbuild/mtest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 323973695..b90a65d27 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -544,9 +544,7 @@ def write_json_log(jsonlogfile: T.TextIO, test_name: str, result: TestRun) -> No jsonlogfile.write(json.dumps(jresult) + '\n') def run_with_mono(fname: str) -> bool: - if fname.endswith('.exe') and not (is_windows() or is_cygwin()): - return True - return False + return fname.endswith('.exe') and not (is_windows() or is_cygwin()) def load_benchmarks(build_dir: str) -> T.List['TestSerialisation']: datafile = Path(build_dir) / 'meson-private' / 'meson_benchmark_setup.dat' From fe46515630a9ec8da23fea3f9940302815385119 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:30:37 -0700 Subject: [PATCH 2/9] mtest: use argparse.type to simplify some code --- mesonbuild/mtest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index b90a65d27..846b474df 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -93,7 +93,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='List available tests.') parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args, help='wrapper to run tests with (e.g. Valgrind)') - parser.add_argument('-C', default='.', dest='wd', + parser.add_argument('-C', default='.', dest='wd', type=os.path.abspath, help='directory to cd into before running') parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', help='Only run tests belonging to the given suite.') @@ -1160,7 +1160,6 @@ def run(options: argparse.Namespace) -> int: if not exe.found(): print('Could not find requested program: {!r}'.format(check_bin)) return 1 - options.wd = os.path.abspath(options.wd) if not options.list and not options.no_rebuild: if not rebuild_all(options.wd): From 793c3d706e81ff639db1a1314b773b0cfdebd7db Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:35:55 -0700 Subject: [PATCH 3/9] backends/backends: sort and cleanup imports --- mesonbuild/backend/backends.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 31ddfb457..774764d62 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -12,23 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, pickle, re +from collections import OrderedDict +from functools import lru_cache +import json +import os +import pickle +import re +import shlex +import subprocess import textwrap +import typing as T + from .. import build from .. import dependencies from .. import mesonlib from .. import mlog -import json -import subprocess -from ..mesonlib import MachineChoice, MesonException, OrderedSet, OptionOverrideProxy -from ..mesonlib import classify_unity_sources, unholder -from ..mesonlib import File from ..compilers import CompilerArgs, VisualStudioLikeCompiler from ..interpreter import Interpreter -from collections import OrderedDict -import shlex -from functools import lru_cache -import typing as T +from ..mesonlib import ( + File, MachineChoice, MesonException, OrderedSet, OptionOverrideProxy, + classify_unity_sources, unholder +) class CleanTrees: From c2a4474b582fb98bd81c0babd1056eeb51d0f1ce Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:39:36 -0700 Subject: [PATCH 4/9] build: cleanup and sort imports --- mesonbuild/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index c20026145..b95988e1c 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy, os, re from collections import OrderedDict, defaultdict -import itertools, pathlib +from functools import lru_cache +import copy import hashlib +import itertools, pathlib +import os import pickle -from functools import lru_cache +import re import typing as T from . import environment From 28e3ce67ae49494d57372f27b6f91580656f77a7 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:54:46 -0700 Subject: [PATCH 5/9] Convert test protocol into an enum This gives us better type safety, and will be important as we add more test methods --- mesonbuild/backend/backends.py | 30 ++++++++++++++++++++++++++---- mesonbuild/interpreter.py | 3 ++- mesonbuild/mintro.py | 2 +- mesonbuild/mtest.py | 5 +++-- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 774764d62..ecdf330b6 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -14,6 +14,7 @@ from collections import OrderedDict from functools import lru_cache +import enum import json import os import pickle @@ -28,12 +29,33 @@ from .. import dependencies from .. import mesonlib from .. import mlog from ..compilers import CompilerArgs, VisualStudioLikeCompiler -from ..interpreter import Interpreter from ..mesonlib import ( File, MachineChoice, MesonException, OrderedSet, OptionOverrideProxy, classify_unity_sources, unholder ) +if T.TYPE_CHECKING: + from ..interpreter import Interpreter + + +class TestProtocol(enum.Enum): + + EXITCODE = 0 + TAP = 1 + + @classmethod + def from_str(cls, string: str) -> 'TestProtocol': + if string == 'exitcode': + return cls.EXITCODE + elif string == 'tap': + return cls.TAP + raise MesonException('unknown test format {}'.format(string)) + + def __str__(self) -> str: + if self is self.EXITCODE: + return 'exitcode' + return 'tap' + class CleanTrees: ''' @@ -91,7 +113,7 @@ class TestSerialisation: needs_exe_wrapper: bool, is_parallel: bool, cmd_args: T.List[str], env: build.EnvironmentVariables, should_fail: bool, timeout: T.Optional[int], workdir: T.Optional[str], - extra_paths: T.List[str], protocol: str, priority: int): + extra_paths: T.List[str], protocol: TestProtocol, priority: int): self.name = name self.project_name = project self.suite = suite @@ -111,7 +133,7 @@ class TestSerialisation: self.priority = priority self.needs_exe_wrapper = needs_exe_wrapper -def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, interpreter: T.Optional[Interpreter] = None) -> T.Optional['Backend']: +def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, interpreter: T.Optional['Interpreter'] = None) -> T.Optional['Backend']: if backend == 'ninja': from . import ninjabackend return ninjabackend.NinjaBackend(build, interpreter) @@ -138,7 +160,7 @@ def get_backend_from_name(backend: str, build: T.Optional[build.Build] = None, i # This class contains the basic functionality that is needed by all backends. # Feel free to move stuff in and out of it as you see fit. class Backend: - def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional[Interpreter]): + def __init__(self, build: T.Optional[build.Build], interpreter: T.Optional['Interpreter']): # Make it possible to construct a dummy backend # This is used for introspection without a build directory if build is None: diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index dd1e57bad..7b8ca63c2 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -33,6 +33,7 @@ from .interpreterbase import FeatureNew, FeatureDeprecated, FeatureNewKwargs from .interpreterbase import ObjectHolder from .modules import ModuleReturnValue from .cmake import CMakeInterpreter +from .backend.backends import TestProtocol from pathlib import Path, PurePath import os @@ -979,7 +980,7 @@ class Test(InterpreterObject): self.should_fail = should_fail self.timeout = timeout self.workdir = workdir - self.protocol = protocol + self.protocol = TestProtocol.from_str(protocol) self.priority = priority def get_exe(self): diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index d5516d454..54e302bc7 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -328,7 +328,7 @@ def get_test_list(testdata) -> T.List[T.Dict[str, T.Union[str, int, T.List[str], to['suite'] = t.suite to['is_parallel'] = t.is_parallel to['priority'] = t.priority - to['protocol'] = t.protocol + to['protocol'] = str(t.protocol) result.append(to) return result diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 846b474df..69da40063 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -43,6 +43,7 @@ from . import environment from . import mlog from .dependencies import ExternalProgram from .mesonlib import MesonException, get_wine_shortpath, split_args +from .backend.backends import TestProtocol if T.TYPE_CHECKING: from .backend.backends import TestSerialisation @@ -631,7 +632,7 @@ class SingleTestRunner: if not self.options.verbose: stdout = tempfile.TemporaryFile("wb+") stderr = tempfile.TemporaryFile("wb+") if self.options.split else stdout - if self.test.protocol == 'tap' and stderr is stdout: + if self.test.protocol is TestProtocol.TAP and stderr is stdout: stdout = tempfile.TemporaryFile("wb+") # Let gdb handle ^C instead of us @@ -741,7 +742,7 @@ class SingleTestRunner: if timed_out: return TestRun(self.test, self.test_env, TestResult.TIMEOUT, [], p.returncode, starttime, duration, stdo, stde, cmd) else: - if self.test.protocol == 'exitcode': + if self.test.protocol is TestProtocol.EXITCODE: return TestRun.make_exitcode(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd) else: if self.options.verbose: From 0c51762463abd72526ac84f3cfeaa286186ae1d7 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 13:59:49 -0700 Subject: [PATCH 6/9] backend/backends: Fix type annotation --- mesonbuild/backend/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index ecdf330b6..ad010115a 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -109,7 +109,7 @@ class ExecutableSerialisation: class TestSerialisation: def __init__(self, name: str, project: str, suite: str, fname: T.List[str], - is_cross_built: bool, exe_wrapper: T.Optional[build.Executable], + is_cross_built: bool, exe_wrapper: T.Optional[dependencies.ExternalProgram], needs_exe_wrapper: bool, is_parallel: bool, cmd_args: T.List[str], env: build.EnvironmentVariables, should_fail: bool, timeout: T.Optional[int], workdir: T.Optional[str], From 083c5f635741a29f93f95c817601dbc66207699d Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 30 Apr 2020 15:36:17 -0700 Subject: [PATCH 7/9] Add native support for gtest tests Gtest can output junit results with a command line switch. We can parse this to get more detailed results than the returncode, and put those in our own Junit output. We basically just throw away the top level 'testsuites' object, then fixup the names of the tests, and shove that into our junit. --- docs/markdown/Reference-manual.md | 13 +++--- docs/markdown/snippets/gtest_protocol.md | 6 +++ mesonbuild/backend/backends.py | 5 +++ mesonbuild/interpreter.py | 6 ++- mesonbuild/mtest.py | 54 ++++++++++++++++++++--- run_unittests.py | 10 +++-- test cases/frameworks/2 gtest/meson.build | 4 +- 7 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 docs/markdown/snippets/gtest_protocol.md diff --git a/docs/markdown/Reference-manual.md b/docs/markdown/Reference-manual.md index 963af9d65..15a438b75 100644 --- a/docs/markdown/Reference-manual.md +++ b/docs/markdown/Reference-manual.md @@ -1662,11 +1662,14 @@ test(..., env: nomalloc, ...) before test is executed even if they have `build_by_default : false`. Since 0.46.0 -- `protocol` specifies how the test results are parsed and can be one - of `exitcode` (the executable's exit code is used by the test harness - to record the outcome of the test) or `tap` ([Test Anything - Protocol](https://www.testanything.org/)). For more on the Meson test - harness protocol read [Unit Tests](Unit-tests.md). Since 0.50.0 +- `protocol` *(Since 0.50.0)* specifies how the test results are parsed and can + be one of `exitcode`, `tap`, or `gtest`. For more information about test + harness protocol read [Unit Tests](Unit-tests.md). The following values are + accepted: + - `exitcode`: the executable's exit code is used by the test harness + to record the outcome of the test) + - `tap` ([Test Anything Protocol](https://www.testanything.org/)) + - `gtest`. *(Since 0.55.0)* for Google Tests. - `priority` specifies the priority of a test. Tests with a higher priority are *started* before tests with a lower priority. diff --git a/docs/markdown/snippets/gtest_protocol.md b/docs/markdown/snippets/gtest_protocol.md new file mode 100644 index 000000000..14f3af915 --- /dev/null +++ b/docs/markdown/snippets/gtest_protocol.md @@ -0,0 +1,6 @@ +## Test protocol for gtest + +Due to the popularity of Gtest (google test) among C and C++ developers meson +now supports a special protocol for gtest. With this protocol meson injects +arguments to gtests to output JUnit, reads that JUnit, and adds the output to +the JUnit it generates. diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index ad010115a..d41cef15e 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -42,6 +42,7 @@ class TestProtocol(enum.Enum): EXITCODE = 0 TAP = 1 + GTEST = 2 @classmethod def from_str(cls, string: str) -> 'TestProtocol': @@ -49,11 +50,15 @@ class TestProtocol(enum.Enum): return cls.EXITCODE elif string == 'tap': return cls.TAP + elif string == 'gtest': + return cls.GTEST raise MesonException('unknown test format {}'.format(string)) def __str__(self) -> str: if self is self.EXITCODE: return 'exitcode' + elif self is self.GTEST: + return 'gtest' return 'tap' diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 7b8ca63c2..c0be92a7e 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -3772,6 +3772,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self @FeatureNewKwargs('test', '0.52.0', ['priority']) @permittedKwargs(permitted_kwargs['test']) def func_test(self, node, args, kwargs): + if kwargs.get('protocol') == 'gtest': + FeatureNew('"gtest" protocol for tests', '0.55.0').use(self.subproject) self.add_test(node, args, kwargs, True) def unpack_env_kwarg(self, kwargs) -> build.EnvironmentVariables: @@ -3823,8 +3825,8 @@ This will become a hard error in the future.''' % kwargs['input'], location=self if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') protocol = kwargs.get('protocol', 'exitcode') - if protocol not in ('exitcode', 'tap'): - raise InterpreterException('Protocol must be "exitcode" or "tap".') + if protocol not in {'exitcode', 'tap', 'gtest'}: + raise InterpreterException('Protocol must be "exitcode", "tap", or "gtest".') suite = [] prj = self.subproject if self.is_subproject() else self.build.project_name for s in mesonlib.stringlistify(kwargs.get('suite', '')): diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 69da40063..4592c90a4 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -94,7 +94,10 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='List available tests.') parser.add_argument('--wrapper', default=None, dest='wrapper', type=split_args, help='wrapper to run tests with (e.g. Valgrind)') - parser.add_argument('-C', default='.', dest='wd', type=os.path.abspath, + parser.add_argument('-C', default='.', dest='wd', + # https://github.com/python/typeshed/issues/3107 + # https://github.com/python/mypy/issues/7177 + type=os.path.abspath, # type: ignore help='directory to cd into before running') parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', help='Only run tests belonging to the given suite.') @@ -349,6 +352,19 @@ class JunitBuilder: def log(self, name: str, test: 'TestRun') -> None: """Log a single test case.""" + if test.junit is not None: + for suite in test.junit.findall('.//testsuite'): + # Assume that we don't need to merge anything here... + suite.attrib['name'] = '{}.{}.{}'.format(test.project, name, suite.attrib['name']) + + # GTest can inject invalid attributes + for case in suite.findall('.//testcase[@result]'): + del case.attrib['result'] + for case in suite.findall('.//testcase[@timestamp]'): + del case.attrib['timestamp'] + self.root.append(suite) + return + # In this case we have a test binary with multiple results. # We want to record this so that each result is recorded # separately @@ -429,11 +445,25 @@ class JunitBuilder: class TestRun: + @classmethod + def make_gtest(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], + returncode: int, starttime: float, duration: float, + stdo: T.Optional[str], stde: T.Optional[str], + cmd: T.Optional[T.List[str]]) -> 'TestRun': + filename = '{}.xml'.format(test.name) + if test.workdir: + filename = os.path.join(test.workdir, filename) + tree = et.parse(filename) + + return cls.make_exitcode( + test, test_env, returncode, starttime, duration, stdo, stde, cmd, + junit=tree) + @classmethod def make_exitcode(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], returncode: int, starttime: float, duration: float, stdo: T.Optional[str], stde: T.Optional[str], - cmd: T.Optional[T.List[str]]) -> 'TestRun': + cmd: T.Optional[T.List[str]], **kwargs) -> 'TestRun': if returncode == GNU_SKIP_RETURNCODE: res = TestResult.SKIP elif returncode == GNU_ERROR_RETURNCODE: @@ -442,15 +472,15 @@ class TestRun: res = TestResult.EXPECTEDFAIL if bool(returncode) else TestResult.UNEXPECTEDPASS else: res = TestResult.FAIL if bool(returncode) else TestResult.OK - return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd) + return cls(test, test_env, res, [], returncode, starttime, duration, stdo, stde, cmd, **kwargs) @classmethod def make_tap(cls, test: 'TestSerialisation', test_env: T.Dict[str, str], returncode: int, starttime: float, duration: float, stdo: str, stde: str, cmd: T.Optional[T.List[str]]) -> 'TestRun': - res = None # T.Optional[TestResult] - results = [] # T.List[TestResult] + res = None # type: T.Optional[TestResult] + results = [] # type: T.List[TestResult] failed = False for i in TAPParser(io.StringIO(stdo)).parse(): @@ -486,7 +516,7 @@ class TestRun: res: TestResult, results: T.List[TestResult], returncode: int, starttime: float, duration: float, stdo: T.Optional[str], stde: T.Optional[str], - cmd: T.Optional[T.List[str]]): + cmd: T.Optional[T.List[str]], *, junit: T.Optional[et.ElementTree] = None): assert isinstance(res, TestResult) self.res = res self.results = results # May be an empty list @@ -499,6 +529,7 @@ class TestRun: self.env = test_env self.should_fail = test.should_fail self.project = test.project_name + self.junit = junit def get_log(self) -> str: res = '--- command ---\n' @@ -652,7 +683,14 @@ class SingleTestRunner: # errors avoid not being able to use the terminal. os.setsid() # type: ignore - p = subprocess.Popen(cmd, + extra_cmd = [] # type: T.List[str] + if self.test.protocol is TestProtocol.GTEST: + gtestname = '{}.xml'.format(self.test.name) + if self.test.workdir: + gtestname = '{}:{}'.format(self.test.workdir, self.test.name) + extra_cmd.append('--gtest_output=xml:{}'.format(gtestname)) + + p = subprocess.Popen(cmd + extra_cmd, stdout=stdout, stderr=stderr, env=self.env, @@ -744,6 +782,8 @@ class SingleTestRunner: else: if self.test.protocol is TestProtocol.EXITCODE: return TestRun.make_exitcode(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd) + elif self.test.protocol is TestProtocol.GTEST: + return TestRun.make_gtest(self.test, self.test_env, p.returncode, starttime, duration, stdo, stde, cmd) else: if self.options.verbose: print(stdo, end='') diff --git a/run_unittests.py b/run_unittests.py index da898a31b..3826762d4 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -4625,8 +4625,7 @@ recommended as it is not supported on some platforms''') schema = et.XMLSchema(et.parse(str(Path(__file__).parent / 'data' / 'schema.xsd'))) - testdir = os.path.join(self.common_test_dir, case) - self.init(testdir) + self.init(case) self.run_tests() junit = et.parse(str(Path(self.builddir) / 'meson-logs' / 'testlog.junit.xml')) @@ -4636,10 +4635,13 @@ recommended as it is not supported on some platforms''') self.fail(e.error_log) def test_junit_valid_tap(self): - self._test_junit('213 tap tests') + self._test_junit(os.path.join(self.common_test_dir, '213 tap tests')) def test_junit_valid_exitcode(self): - self._test_junit('44 test args') + self._test_junit(os.path.join(self.common_test_dir, '44 test args')) + + def test_junit_valid_gtest(self): + self._test_junit(os.path.join(self.framework_test_dir, '2 gtest')) class FailureTests(BasePlatformTests): diff --git a/test cases/frameworks/2 gtest/meson.build b/test cases/frameworks/2 gtest/meson.build index 2d93b5294..ea3ef48e8 100644 --- a/test cases/frameworks/2 gtest/meson.build +++ b/test cases/frameworks/2 gtest/meson.build @@ -8,7 +8,7 @@ endif gtest_nomain = dependency('gtest', main : false, method : 'system') e = executable('testprog', 'test.cc', dependencies : gtest) -test('gtest test', e) +test('gtest test', e, protocol : 'gtest') e = executable('testprog_nomain', 'test_nomain.cc', dependencies : gtest_nomain) -test('gtest nomain test', e) +test('gtest nomain test', e, protocol : 'gtest') From bbbfccf0af26fd6f3b3970d6d9452ba8acd13128 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 1 May 2020 09:59:56 -0700 Subject: [PATCH 8/9] docs: Reformat unit-tests to be ~80 characters per line --- docs/markdown/Unit-tests.md | 125 ++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 41 deletions(-) diff --git a/docs/markdown/Unit-tests.md b/docs/markdown/Unit-tests.md index 0785549a5..b1f4cffeb 100644 --- a/docs/markdown/Unit-tests.md +++ b/docs/markdown/Unit-tests.md @@ -4,20 +4,24 @@ short-description: Meson's own unit-test system # Unit tests -Meson comes with a fully functional unit test system. To use it simply build an executable and then use it in a test. +Meson comes with a fully functional unit test system. To use it simply build +an executable and then use it in a test. ```meson e = executable('prog', 'testprog.c') test('name of test', e) ``` -You can add as many tests as you want. They are run with the command `ninja test`. +You can add as many tests as you want. They are run with the command `ninja +test`. -Meson captures the output of all tests and writes it in the log file `meson-logs/testlog.txt`. +Meson captures the output of all tests and writes it in the log file +`meson-logs/testlog.txt`. ## Test parameters -Some tests require the use of command line arguments or environment variables. These are simple to define. +Some tests require the use of command line arguments or environment +variables. These are simple to define. ```meson test('command line test', exe, args : ['first', 'second']) @@ -29,38 +33,46 @@ Note how you need to specify multiple values as an array. ### MALLOC_PERTURB_ By default, environment variable -[`MALLOC_PERTURB_`](http://man7.org/linux/man-pages/man3/mallopt.3.html) -is set to a random value between 1..255. This can help find memory -leaks on configurations using glibc, including with non-GCC compilers. -This feature can be disabled as discussed in [test()](Reference-manual.md#test). +[`MALLOC_PERTURB_`](http://man7.org/linux/man-pages/man3/mallopt.3.html) is +set to a random value between 1..255. This can help find memory leaks on +configurations using glibc, including with non-GCC compilers. This feature +can be disabled as discussed in [test()](Reference-manual.md#test). ## Coverage If you enable coverage measurements by giving Meson the command line flag -`-Db_coverage=true`, you can generate coverage reports after running the tests -(running the tests is required to gather the list of functions that get -called). Meson will autodetect what coverage generator tools you have installed -and will generate the corresponding targets. These targets are `coverage-xml` -and `coverage-text` which are both provided by [Gcovr](http://gcovr.com) -(version 3.3 or higher) and `coverage-html`, which requires -[Lcov](https://ltp.sourceforge.io/coverage/lcov.php) and -[GenHTML](https://linux.die.net/man/1/genhtml) or -[Gcovr](http://gcovr.com). As a convenience, a high-level `coverage` target is -also generated which will produce all 3 coverage report types, if possible. - -The output of these commands is written to the log directory `meson-logs` in your build directory. +`-Db_coverage=true`, you can generate coverage reports after running the +tests (running the tests is required to gather the list of functions that get +called). Meson will autodetect what coverage generator tools you have +installed and will generate the corresponding targets. These targets are +`coverage-xml` and `coverage-text` which are both provided by +[Gcovr](http://gcovr.com) (version 3.3 or higher) and `coverage-html`, which +requires [Lcov](https://ltp.sourceforge.io/coverage/lcov.php) and +[GenHTML](https://linux.die.net/man/1/genhtml) or [Gcovr](http://gcovr.com). +As a convenience, a high-level `coverage` target is also generated which will +produce all 3 coverage report types, if possible. + +The output of these commands is written to the log directory `meson-logs` in +your build directory. ## Parallelism -To reduce test times, Meson will by default run multiple unit tests in parallel. It is common to have some tests which can not be run in parallel because they require unique hold on some resource such as a file or a D-Bus name. You have to specify these tests with a keyword argument. +To reduce test times, Meson will by default run multiple unit tests in +parallel. It is common to have some tests which can not be run in parallel +because they require unique hold on some resource such as a file or a D-Bus +name. You have to specify these tests with a keyword argument. ```meson test('unique test', t, is_parallel : false) ``` -Meson will then make sure that no other unit test is running at the same time. Non-parallel tests take longer to run so it is recommended that you write your unit tests to be parallel executable whenever possible. +Meson will then make sure that no other unit test is running at the same +time. Non-parallel tests take longer to run so it is recommended that you +write your unit tests to be parallel executable whenever possible. -By default Meson uses as many concurrent processes as there are cores on the test machine. You can override this with the environment variable `MESON_TESTTHREADS` like this. +By default Meson uses as many concurrent processes as there are cores on the +test machine. You can override this with the environment variable +`MESON_TESTTHREADS` like this. ```console $ MESON_TESTTHREADS=5 ninja test @@ -70,7 +82,10 @@ $ MESON_TESTTHREADS=5 ninja test *(added in version 0.52.0)* -Tests can be assigned a priority that determines when a test is *started*. Tests with higher priority are started first, tests with lower priority started later. The default priority is 0, meson makes no guarantee on the ordering of tests with identical priority. +Tests can be assigned a priority that determines when a test is *started*. +Tests with higher priority are started first, tests with lower priority +started later. The default priority is 0, meson makes no guarantee on the +ordering of tests with identical priority. ```meson test('started second', t, priority : 0) @@ -78,23 +93,37 @@ test('started third', t, priority : -50) test('started first', t, priority : 1000) ``` -Note that the test priority only affects the starting order of tests and subsequent tests are affected by how long it takes previous tests to complete. It is thus possible that a higher-priority test is still running when lower-priority tests with a shorter runtime have completed. +Note that the test priority only affects the starting order of tests and +subsequent tests are affected by how long it takes previous tests to +complete. It is thus possible that a higher-priority test is still running +when lower-priority tests with a shorter runtime have completed. ## Skipped tests and hard errors Sometimes a test can only determine at runtime that it can not be run. -For the default `exitcode` testing protocol, the GNU standard approach in this case is to exit the program with error code 77. Meson will detect this and report these tests as skipped rather than failed. This behavior was added in version 0.37.0. +For the default `exitcode` testing protocol, the GNU standard approach in +this case is to exit the program with error code 77. Meson will detect this +and report these tests as skipped rather than failed. This behavior was added +in version 0.37.0. -For TAP-based tests, skipped tests should print a single line starting with `1..0 # SKIP`. +For TAP-based tests, skipped tests should print a single line starting with +`1..0 # SKIP`. -In addition, sometimes a test fails set up so that it should fail even if it is marked as an expected failure. The GNU standard approach in this case is to exit the program with error code 99. Again, Meson will detect this and report these tests as `ERROR`, ignoring the setting of `should_fail`. This behavior was added in version 0.50.0. +In addition, sometimes a test fails set up so that it should fail even if it +is marked as an expected failure. The GNU standard approach in this case is +to exit the program with error code 99. Again, Meson will detect this and +report these tests as `ERROR`, ignoring the setting of `should_fail`. This +behavior was added in version 0.50.0. ## Testing tool -The goal of the meson test tool is to provide a simple way to run tests in a variety of different ways. The tool is designed to be run in the build directory. +The goal of the meson test tool is to provide a simple way to run tests in a +variety of different ways. The tool is designed to be run in the build +directory. -The simplest thing to do is just to run all tests, which is equivalent to running `ninja test`. +The simplest thing to do is just to run all tests, which is equivalent to +running `ninja test`. ```console $ meson test @@ -125,7 +154,8 @@ Tests belonging to a suite `suite` can be run as follows $ meson test --suite (sub)project_name:suite ``` -Since version *0.46*, `(sub)project_name` can be omitted if it is the top-level project. +Since version *0.46*, `(sub)project_name` can be omitted if it is the +top-level project. Multiple suites are specified like: @@ -145,7 +175,8 @@ Sometimes you need to run the tests multiple times, which is done like this: $ meson test --repeat=10 ``` -Invoking tests via a helper executable such as Valgrind can be done with the `--wrap` argument +Invoking tests via a helper executable such as Valgrind can be done with the +`--wrap` argument ```console $ meson test --wrap=valgrind testname @@ -163,17 +194,25 @@ Meson also supports running the tests under GDB. Just doing this: $ meson test --gdb testname ``` -Meson will launch `gdb` all set up to run the test. Just type `run` in the GDB command prompt to start the program. +Meson will launch `gdb` all set up to run the test. Just type `run` in the +GDB command prompt to start the program. -The second use case is a test that segfaults only rarely. In this case you can invoke the following command: +The second use case is a test that segfaults only rarely. In this case you +can invoke the following command: ```console $ meson test --gdb --repeat=10000 testname ``` -This runs the test up to 10 000 times under GDB automatically. If the program crashes, GDB will halt and the user can debug the application. Note that testing timeouts are disabled in this case so `meson test` will not kill `gdb` while the developer is still debugging it. The downside is that if the test binary freezes, the test runner will wait forever. +This runs the test up to 10 000 times under GDB automatically. If the program +crashes, GDB will halt and the user can debug the application. Note that +testing timeouts are disabled in this case so `meson test` will not kill +`gdb` while the developer is still debugging it. The downside is that if the +test binary freezes, the test runner will wait forever. -Sometimes, the GDB binary is not in the PATH variable or the user wants to use a GDB replacement. Therefore, the invoked GDB program can be specified *(added 0.52.0)*: +Sometimes, the GDB binary is not in the PATH variable or the user wants to +use a GDB replacement. Therefore, the invoked GDB program can be specified +*(added 0.52.0)*: ```console $ meson test --gdb --gdb-path /path/to/gdb testname @@ -183,12 +222,16 @@ $ meson test --gdb --gdb-path /path/to/gdb testname $ meson test --print-errorlogs ``` -Meson will report the output produced by the failing tests along with other useful information as the environmental variables. This is useful, for example, when you run the tests on Travis-CI, Jenkins and the like. +Meson will report the output produced by the failing tests along with other +useful information as the environmental variables. This is useful, for +example, when you run the tests on Travis-CI, Jenkins and the like. -For further information see the command line help of Meson by running `meson test -h`. +For further information see the command line help of Meson by running `meson +test -h`. ## Legacy notes -If `meson test` does not work for you, you likely have a old version of Meson. -In that case you should call `mesontest` instead. If `mesontest` doesn't work -either you have a very old version prior to 0.37.0 and should upgrade. +If `meson test` does not work for you, you likely have a old version of +Meson. In that case you should call `mesontest` instead. If `mesontest` +doesn't work either you have a very old version prior to 0.37.0 and should +upgrade. From dbe00dfe95e9630bd733a45f32076bab6ff80226 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 1 May 2020 10:07:30 -0700 Subject: [PATCH 9/9] docs/unit-tests: Add information about output files --- docs/markdown/Unit-tests.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/markdown/Unit-tests.md b/docs/markdown/Unit-tests.md index b1f4cffeb..bd91dbb6b 100644 --- a/docs/markdown/Unit-tests.md +++ b/docs/markdown/Unit-tests.md @@ -235,3 +235,28 @@ If `meson test` does not work for you, you likely have a old version of Meson. In that case you should call `mesontest` instead. If `mesontest` doesn't work either you have a very old version prior to 0.37.0 and should upgrade. + +## Test outputs + +Meson will write several different files with detailed results of running +tests. These will be written into $builddir/meson-logs/ + +### testlog.json + +This is not a proper json file, but a file containing one valid json object +per line. This is file is designed so each line is streamed out as each test +is run, so it can be read as a stream while the test harness is running + +### testlog.junit.xml + +This is a valid JUnit XML description of all tests run. It is not streamed +out, and is written only once all tests complete running. + +When tests use the `tap` protocol each test will be recorded as a testsuite +container, with each case named by the number of the result. + +When tests use the `gtest` protocol meson will inject arguments to the test +to generate it's own JUnit XML, which meson will include as part of this XML +file. + +*New in 0.55.0*