diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 449afe74a..f895531e6 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -98,6 +98,7 @@ class Build: self.dep_manifest_name = None self.dep_manifest = {} self.cross_stdlibs = {} + self.test_setups = {} def add_compiler(self, compiler): if self.static_linker is None and compiler.needs_static_linker(): @@ -1507,3 +1508,10 @@ class RunScript(dict): assert(isinstance(args, list)) self['exe'] = script self['args'] = args + +class TestSetup: + def __init__(self, *, exe_wrapper=None, gdb=None, timeout_multiplier=None, env=None): + self.exe_wrapper = exe_wrapper + self.gdb = gdb + self.timeout_multiplier = timeout_multiplier + self.env = env diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index ac1540136..eda6f0746 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -30,6 +30,7 @@ from .interpreterbase import InterpreterException, InvalidArguments, InvalidCode from .interpreterbase import InterpreterObject, MutableInterpreterObject import os, sys, shutil, uuid +import re import importlib @@ -1207,6 +1208,7 @@ class Interpreter(InterpreterBase): 'add_project_arguments': self.func_add_project_arguments, 'add_global_link_arguments': self.func_add_global_link_arguments, 'add_project_link_arguments': self.func_add_project_link_arguments, + 'add_test_setup': self.func_add_test_setup, 'add_languages': self.func_add_languages, 'find_program': self.func_find_program, 'find_library': self.func_find_library, @@ -1942,22 +1944,7 @@ requirements use the version keyword argument instead.''') def func_test(self, node, args, kwargs): self.add_test(node, args, kwargs, True) - def add_test(self, node, args, kwargs, is_base_test): - if len(args) != 2: - raise InterpreterException('Incorrect number of arguments') - if not isinstance(args[0], str): - raise InterpreterException('First argument of test must be a string.') - if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)): - raise InterpreterException('Second argument must be executable.') - par = kwargs.get('is_parallel', True) - if not isinstance(par, bool): - raise InterpreterException('Keyword argument is_parallel must be a boolean.') - cmd_args = kwargs.get('args', []) - if not isinstance(cmd_args, list): - cmd_args = [cmd_args] - for i in cmd_args: - if not isinstance(i, (str, mesonlib.File)): - raise InterpreterException('Command line arguments must be strings') + def unpack_env_kwarg(self, kwargs): envlist = kwargs.get('env', []) if isinstance(envlist, EnvironmentVariablesHolder): env = envlist.held_object @@ -1974,8 +1961,25 @@ requirements use the version keyword argument instead.''') if ' ' in k: raise InterpreterException('Env var key must not have spaces in it.') env[k] = val - if not isinstance(envlist, list): - envlist = [envlist] + return env + + def add_test(self, node, args, kwargs, is_base_test): + if len(args) != 2: + raise InterpreterException('Incorrect number of arguments') + if not isinstance(args[0], str): + raise InterpreterException('First argument of test must be a string.') + if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)): + raise InterpreterException('Second argument must be executable.') + par = kwargs.get('is_parallel', True) + if not isinstance(par, bool): + raise InterpreterException('Keyword argument is_parallel must be a boolean.') + cmd_args = kwargs.get('args', []) + if not isinstance(cmd_args, list): + cmd_args = [cmd_args] + for i in cmd_args: + if not isinstance(i, (str, mesonlib.File)): + raise InterpreterException('Command line arguments must be strings') + env = self.unpack_env_kwarg(kwargs) should_fail = kwargs.get('should_fail', False) if not isinstance(should_fail, bool): raise InterpreterException('Keyword argument should_fail must be a boolean.') @@ -2138,6 +2142,47 @@ requirements use the version keyword argument instead.''') i = IncludeDirsHolder(build.IncludeDirs(self.subdir, args, is_system)) return i + @stringArgs + def func_add_test_setup(self, node, args, kwargs): + if len(args) != 1: + raise InterpreterException('Add_test_setup needs one argument for the setup name.') + setup_name = args[0] + if re.fullmatch('[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None: + raise InterpreterException('Setup name may only contain alphanumeric characters.') + try: + inp = kwargs.get('exe_wrapper', []) + if not isinstance(inp, list): + inp = [inp] + exe_wrapper = [] + for i in inp: + if hasattr(i, 'held_object'): + i = i.held_object + if isinstance(i, str): + exe_wrapper.append(i) + elif isinstance(i, dependencies.ExternalProgram): + if not i.found(): + raise InterpreterException('Tried to use non-found external executable.') + exe_wrapper += i.get_command() + else: + raise InterpreterException('Exe wrapper can only contain strings or external binaries.') + except KeyError: + exe_wrapper = None + gdb = kwargs.get('gdb', False) + if not isinstance(gdb, bool): + raise InterpreterException('Gdb option must be a boolean') + timeout_multiplier = kwargs.get('timeout_multiplier', 1) + if not isinstance(timeout_multiplier, int): + raise InterpreterException('Timeout multiplier must be a number.') + env = self.unpack_env_kwarg(kwargs) + setupobj = build.TestSetup(exe_wrapper=exe_wrapper, + gdb=gdb, + timeout_multiplier=timeout_multiplier, + env=env) + if self.subproject == '': + # Dunno what we should do with subprojects really. Let's start simple + # and just use the master project ones. + self.build.test_setups[setup_name] = setupobj + @stringArgs def func_add_global_arguments(self, node, args, kwargs): if self.subproject != '': diff --git a/mesontest.py b/mesontest.py index ecf1b41b2..c7c60c0c9 100755 --- a/mesontest.py +++ b/mesontest.py @@ -69,7 +69,7 @@ parser.add_argument('--suite', default=None, dest='suite', parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', help='Do not split stderr and stdout in test logs.') parser.add_argument('--print-errorlogs', default=False, action='store_true', - help="Whether to print faling tests' logs.") + help="Whether to print failing tests' logs.") parser.add_argument('--benchmark', default=False, action='store_true', help="Run benchmarks instead of tests.") parser.add_argument('--logbase', default='testlog', @@ -82,6 +82,8 @@ parser.add_argument('-t', '--timeout-multiplier', type=float, default=1.0, help='Define a multiplier for test timeout, for example ' ' when running tests in particular conditions they might take' ' more time to execute.') +parser.add_argument('--setup', default=None, dest='setup', + help='Which test setup to use.') parser.add_argument('args', nargs='*') class TestRun(): @@ -206,6 +208,7 @@ class TestHarness: cmd = wrap + cmd + test.cmd_args starttime = time.time() child_env = os.environ.copy() + child_env.update(self.options.global_env.get_env(child_env)) if isinstance(test.env, build.EnvironmentVariables): test.env = test.env.get_env(child_env) @@ -328,8 +331,12 @@ class TestHarness: logfilename = logfile_base + '.txt' jsonlogfilename = logfile_base + '.json' else: - wrap = self.options.wrapper.split() - namebase = wrap[0] + if isinstance(self.options.wrapper, str): + wrap = self.options.wrapper.split() + else: + wrap = self.options.wrapper + assert(isinstance(wrap, list)) + namebase = os.path.split(wrap[0])[1] logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt' jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json' tests = self.get_tests() @@ -453,11 +460,38 @@ def filter_tests(suite, tests): return [x for x in tests if suite in x.suite] +def merge_suite_options(options): + buildfile = os.path.join(options.wd, 'meson-private/build.dat') + with open(buildfile, 'rb') as f: + build = pickle.load(f) + setups = build.test_setups + if options.setup not in setups: + sys.exit('Unknown test setup: %s' % options.setup) + current = setups[options.setup] + if not options.gdb: + options.gdb = current.gdb + if options.timeout_multiplier is None: + options.timeout_multiplier = current.timeout_multiplier +# if options.env is None: +# options.env = current.env # FIXME, should probably merge options here. + if options.wrapper is not None and current.exe_wrapper is not None: + sys.exit('Conflict: both test setup and command line specify an exe wrapper.') + if options.wrapper is None: + options.wrapper = current.exe_wrapper + return current.env + def run(args): options = parser.parse_args(args) if options.benchmark: options.num_processes = 1 + if options.setup is not None: + global_env = merge_suite_options(options) + else: + global_env = build.EnvironmentVariables() + + setattr(options, 'global_env', global_env) + if options.gdb: options.verbose = True diff --git a/run_unittests.py b/run_unittests.py index 179bed690..73268003e 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -57,12 +57,14 @@ class LinuxlikeTests(unittest.TestCase): src_root = os.path.dirname(__file__) src_root = os.path.join(os.getcwd(), src_root) self.builddir = tempfile.mkdtemp() + self.logdir = os.path.join(self.builddir, 'meson-logs') self.prefix = '/usr' self.libdir = os.path.join(self.prefix, 'lib') self.installdir = os.path.join(self.builddir, 'install') self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py')] self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')] self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')] + self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir] self.ninja_command = [detect_ninja(), '-C', self.builddir] self.common_test_dir = os.path.join(src_root, 'test cases/common') self.vala_test_dir = os.path.join(src_root, 'test cases/vala') @@ -89,6 +91,9 @@ class LinuxlikeTests(unittest.TestCase): def build(self): self._run(self.ninja_command) + def run_tests(self): + self._run(self.ninja_command + ['test']) + def install(self): os.environ['DESTDIR'] = self.installdir self._run(self.ninja_command + ['install']) @@ -387,6 +392,23 @@ class LinuxlikeTests(unittest.TestCase): meson_exe_dat2 = glob(os.path.join(self.privatedir, 'meson_exe*.dat')) self.assertListEqual(meson_exe_dat1, meson_exe_dat2) + def test_testsetups(self): + if not shutil.which('valgrind'): + raise unittest.SkipTest('Valgrind not installed.') + testdir = os.path.join(self.unit_test_dir, '2 testsetups') + self.init(testdir) + self.build() + self.run_tests() + with open(os.path.join(self.logdir, 'testlog.txt')) as f: + basic_log = f.read() + self.assertRaises(subprocess.CalledProcessError, + self._run, self.mtest_command + ['--setup=valgrind']) + with open(os.path.join(self.logdir, 'testlog-valgrind.txt')) as f: + vg_log = f.read() + self.assertFalse('TEST_ENV is set' in basic_log) + self.assertFalse('Memcheck' in basic_log) + self.assertTrue('TEST_ENV is set' in vg_log) + self.assertTrue('Memcheck' in vg_log) class RewriterTests(unittest.TestCase): @@ -446,5 +468,6 @@ class RewriterTests(unittest.TestCase): self.assertEqual(top, self.read_contents('meson.build')) self.assertEqual(s2, self.read_contents('sub2/meson.build')) + if __name__ == '__main__': unittest.main() diff --git a/test cases/unit/2 testsetups/buggy.c b/test cases/unit/2 testsetups/buggy.c new file mode 100644 index 000000000..5d20a244c --- /dev/null +++ b/test cases/unit/2 testsetups/buggy.c @@ -0,0 +1,14 @@ +#include +#include + +#include + +int main(int argc, char **argv) { + char *ten = malloc(10); + do_nasty(ten); + free(ten); + if(getenv("TEST_ENV")) { + printf("TEST_ENV is set.\n"); + } + return 0; +} diff --git a/test cases/unit/2 testsetups/impl.c b/test cases/unit/2 testsetups/impl.c new file mode 100644 index 000000000..d87f3de51 --- /dev/null +++ b/test cases/unit/2 testsetups/impl.c @@ -0,0 +1,5 @@ +/* Write past the end. */ + +void do_nasty(char *ptr) { + ptr[10] = 'n'; +} diff --git a/test cases/unit/2 testsetups/impl.h b/test cases/unit/2 testsetups/impl.h new file mode 100644 index 000000000..7a08cb329 --- /dev/null +++ b/test cases/unit/2 testsetups/impl.h @@ -0,0 +1,3 @@ +#pragma once + +void do_nasty(char *ptr); diff --git a/test cases/unit/2 testsetups/meson.build b/test cases/unit/2 testsetups/meson.build new file mode 100644 index 000000000..a65548e02 --- /dev/null +++ b/test cases/unit/2 testsetups/meson.build @@ -0,0 +1,16 @@ +project('testsetups', 'c') + +vg = find_program('valgrind') + +# This is only set when running under Valgrind test setup. +env = environment() +env.set('TEST_ENV', '1') + +add_test_setup('valgrind', + exe_wrapper : [vg, '--error-exitcode=1', '--leak-check=full'], + timeout_multiplier : 100, + env : env) + +buggy = executable('buggy', 'buggy.c', 'impl.c') +test('Test buggy', buggy) +