Merge pull request #1254 from mesonbuild/testsetups

Add test setups

mesonbuild/build.py

@@ -98,6 +98,7 @@ class Build:
         self.dep_manifest_name = None
         self.dep_manifest = {}
         self.cross_stdlibs = {}
+        self.test_setups = {}
 
     def add_compiler(self, compiler):
         if self.static_linker is None and compiler.needs_static_linker():
@@ -1507,3 +1508,10 @@ class RunScript(dict):
         assert(isinstance(args, list))
         self['exe'] = script
         self['args'] = args
+
+class TestSetup:
+    def __init__(self, *, exe_wrapper=None, gdb=None, timeout_multiplier=None, env=None):
+        self.exe_wrapper = exe_wrapper
+        self.gdb = gdb
+        self.timeout_multiplier = timeout_multiplier
+        self.env = env

mesonbuild/interpreter.py

@@ -30,6 +30,7 @@ from .interpreterbase import InterpreterException, InvalidArguments, InvalidCode
 from .interpreterbase import InterpreterObject, MutableInterpreterObject
 
 import os, sys, shutil, uuid
+import re
 
 import importlib
 
@@ -1207,6 +1208,7 @@ class Interpreter(InterpreterBase):
                            'add_project_arguments': self.func_add_project_arguments,
                            'add_global_link_arguments': self.func_add_global_link_arguments,
                            'add_project_link_arguments': self.func_add_project_link_arguments,
+                           'add_test_setup': self.func_add_test_setup,
                            'add_languages': self.func_add_languages,
                            'find_program': self.func_find_program,
                            'find_library': self.func_find_library,
@@ -1942,22 +1944,7 @@ requirements use the version keyword argument instead.''')
     def func_test(self, node, args, kwargs):
         self.add_test(node, args, kwargs, True)
 
-    def add_test(self, node, args, kwargs, is_base_test):
-        if len(args) != 2:
-            raise InterpreterException('Incorrect number of arguments')
-        if not isinstance(args[0], str):
-            raise InterpreterException('First argument of test must be a string.')
-        if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)):
-            raise InterpreterException('Second argument must be executable.')
-        par = kwargs.get('is_parallel', True)
-        if not isinstance(par, bool):
-            raise InterpreterException('Keyword argument is_parallel must be a boolean.')
-        cmd_args = kwargs.get('args', [])
-        if not isinstance(cmd_args, list):
-            cmd_args = [cmd_args]
-        for i in cmd_args:
-            if not isinstance(i, (str, mesonlib.File)):
-                raise InterpreterException('Command line arguments must be strings')
+    def unpack_env_kwarg(self, kwargs):
         envlist = kwargs.get('env', [])
         if isinstance(envlist, EnvironmentVariablesHolder):
             env = envlist.held_object
@@ -1974,8 +1961,25 @@ requirements use the version keyword argument instead.''')
                 if ' ' in k:
                     raise InterpreterException('Env var key must not have spaces in it.')
                 env[k] = val
-        if not isinstance(envlist, list):
-            envlist = [envlist]
+        return env
+
+    def add_test(self, node, args, kwargs, is_base_test):
+        if len(args) != 2:
+            raise InterpreterException('Incorrect number of arguments')
+        if not isinstance(args[0], str):
+            raise InterpreterException('First argument of test must be a string.')
+        if not isinstance(args[1], (ExecutableHolder, JarHolder, ExternalProgramHolder)):
+            raise InterpreterException('Second argument must be executable.')
+        par = kwargs.get('is_parallel', True)
+        if not isinstance(par, bool):
+            raise InterpreterException('Keyword argument is_parallel must be a boolean.')
+        cmd_args = kwargs.get('args', [])
+        if not isinstance(cmd_args, list):
+            cmd_args = [cmd_args]
+        for i in cmd_args:
+            if not isinstance(i, (str, mesonlib.File)):
+                raise InterpreterException('Command line arguments must be strings')
+        env = self.unpack_env_kwarg(kwargs)
         should_fail = kwargs.get('should_fail', False)
         if not isinstance(should_fail, bool):
             raise InterpreterException('Keyword argument should_fail must be a boolean.')
@@ -2138,6 +2142,47 @@ requirements use the version keyword argument instead.''')
         i = IncludeDirsHolder(build.IncludeDirs(self.subdir, args, is_system))
         return i
 
+    @stringArgs
+    def func_add_test_setup(self, node, args, kwargs):
+        if len(args) != 1:
+            raise InterpreterException('Add_test_setup needs one argument for the setup name.')
+        setup_name = args[0]
+        if re.fullmatch('[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None:
+            raise InterpreterException('Setup name may only contain alphanumeric characters.')
+        try:
+            inp = kwargs.get('exe_wrapper', [])
+            if not isinstance(inp, list):
+                inp = [inp]
+            exe_wrapper = []
+            for i in inp:
+                if hasattr(i, 'held_object'):
+                    i = i.held_object
+                if isinstance(i, str):
+                    exe_wrapper.append(i)
+                elif isinstance(i, dependencies.ExternalProgram):
+                    if not i.found():
+                        raise InterpreterException('Tried to use non-found external executable.')
+                    exe_wrapper += i.get_command()
+                else:
+                    raise InterpreterException('Exe wrapper can only contain strings or external binaries.')
+        except KeyError:
+            exe_wrapper = None
+        gdb = kwargs.get('gdb', False)
+        if not isinstance(gdb, bool):
+            raise InterpreterException('Gdb option must be a boolean')
+        timeout_multiplier = kwargs.get('timeout_multiplier', 1)
+        if not isinstance(timeout_multiplier, int):
+            raise InterpreterException('Timeout multiplier must be a number.')
+        env = self.unpack_env_kwarg(kwargs)
+        setupobj = build.TestSetup(exe_wrapper=exe_wrapper,
+                                   gdb=gdb,
+                                   timeout_multiplier=timeout_multiplier,
+                                   env=env)
+        if self.subproject == '':
+            # Dunno what we should do with subprojects really. Let's start simple
+            # and just use the master project ones.
+            self.build.test_setups[setup_name] = setupobj
+
     @stringArgs
     def func_add_global_arguments(self, node, args, kwargs):
         if self.subproject != '':

mesontest.py

@@ -69,7 +69,7 @@ parser.add_argument('--suite', default=None, dest='suite',
 parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
                     help='Do not split stderr and stdout in test logs.')
 parser.add_argument('--print-errorlogs', default=False, action='store_true',
-                    help="Whether to print faling tests' logs.")
+                    help="Whether to print failing tests' logs.")
 parser.add_argument('--benchmark', default=False, action='store_true',
                     help="Run benchmarks instead of tests.")
 parser.add_argument('--logbase', default='testlog',
@@ -82,6 +82,8 @@ parser.add_argument('-t', '--timeout-multiplier', type=float, default=1.0,
                     help='Define a multiplier for test timeout, for example '
                     ' when running tests in particular conditions they might take'
                     ' more time to execute.')
+parser.add_argument('--setup', default=None, dest='setup',
+                    help='Which test setup to use.')
 parser.add_argument('args', nargs='*')
 
 class TestRun():
@@ -206,6 +208,7 @@ class TestHarness:
             cmd = wrap + cmd + test.cmd_args
             starttime = time.time()
             child_env = os.environ.copy()
+            child_env.update(self.options.global_env.get_env(child_env))
             if isinstance(test.env, build.EnvironmentVariables):
                 test.env = test.env.get_env(child_env)
 
@@ -328,8 +331,12 @@ class TestHarness:
             logfilename = logfile_base + '.txt'
             jsonlogfilename = logfile_base + '.json'
         else:
-            wrap = self.options.wrapper.split()
-            namebase = wrap[0]
+            if isinstance(self.options.wrapper, str):
+                wrap = self.options.wrapper.split()
+            else:
+                wrap = self.options.wrapper
+            assert(isinstance(wrap, list))
+            namebase = os.path.split(wrap[0])[1]
             logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt'
             jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json'
         tests = self.get_tests()
@@ -453,11 +460,38 @@ def filter_tests(suite, tests):
     return [x for x in tests if suite in x.suite]
 
 
+def merge_suite_options(options):
+    buildfile = os.path.join(options.wd, 'meson-private/build.dat')
+    with open(buildfile, 'rb') as f:
+        build = pickle.load(f)
+    setups = build.test_setups
+    if options.setup not in setups:
+        sys.exit('Unknown test setup: %s' % options.setup)
+    current = setups[options.setup]
+    if not options.gdb:
+        options.gdb = current.gdb
+    if options.timeout_multiplier is None:
+        options.timeout_multiplier = current.timeout_multiplier
+#    if options.env is None:
+#        options.env = current.env # FIXME, should probably merge options here.
+    if options.wrapper is not None and current.exe_wrapper is not None:
+        sys.exit('Conflict: both test setup and command line specify an exe wrapper.')
+    if options.wrapper is None:
+        options.wrapper = current.exe_wrapper
+    return current.env
+
 def run(args):
     options = parser.parse_args(args)
     if options.benchmark:
         options.num_processes = 1
 
+    if options.setup is not None:
+        global_env = merge_suite_options(options)
+    else:
+        global_env = build.EnvironmentVariables()
+
+    setattr(options, 'global_env', global_env)
+
     if options.gdb:
         options.verbose = True
 

run_unittests.py

@@ -57,12 +57,14 @@ class LinuxlikeTests(unittest.TestCase):
         src_root = os.path.dirname(__file__)
         src_root = os.path.join(os.getcwd(), src_root)
         self.builddir = tempfile.mkdtemp()
+        self.logdir = os.path.join(self.builddir, 'meson-logs')
         self.prefix = '/usr'
         self.libdir = os.path.join(self.prefix, 'lib')
         self.installdir = os.path.join(self.builddir, 'install')
         self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py')]
         self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')]
         self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')]
+        self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir]
         self.ninja_command = [detect_ninja(), '-C', self.builddir]
         self.common_test_dir = os.path.join(src_root, 'test cases/common')
         self.vala_test_dir = os.path.join(src_root, 'test cases/vala')
@@ -89,6 +91,9 @@ class LinuxlikeTests(unittest.TestCase):
     def build(self):
         self._run(self.ninja_command)
 
+    def run_tests(self):
+        self._run(self.ninja_command + ['test'])
+
     def install(self):
         os.environ['DESTDIR'] = self.installdir
         self._run(self.ninja_command + ['install'])
@@ -387,6 +392,23 @@ class LinuxlikeTests(unittest.TestCase):
         meson_exe_dat2 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
         self.assertListEqual(meson_exe_dat1, meson_exe_dat2)
 
+    def test_testsetups(self):
+        if not shutil.which('valgrind'):
+                raise unittest.SkipTest('Valgrind not installed.')
+        testdir = os.path.join(self.unit_test_dir, '2 testsetups')
+        self.init(testdir)
+        self.build()
+        self.run_tests()
+        with open(os.path.join(self.logdir, 'testlog.txt')) as f:
+            basic_log = f.read()
+        self.assertRaises(subprocess.CalledProcessError,
+                          self._run, self.mtest_command + ['--setup=valgrind'])
+        with open(os.path.join(self.logdir, 'testlog-valgrind.txt')) as f:
+            vg_log = f.read()
+        self.assertFalse('TEST_ENV is set' in basic_log)
+        self.assertFalse('Memcheck' in basic_log)
+        self.assertTrue('TEST_ENV is set' in vg_log)
+        self.assertTrue('Memcheck' in vg_log)
 
 class RewriterTests(unittest.TestCase):
 
@@ -446,5 +468,6 @@ class RewriterTests(unittest.TestCase):
         self.assertEqual(top, self.read_contents('meson.build'))
         self.assertEqual(s2, self.read_contents('sub2/meson.build'))
 
+
 if __name__ == '__main__':
     unittest.main()

test cases/unit/2 testsetups/buggy.c

@@ -0,0 +1,14 @@
+#include<stdio.h>
+#include<stdlib.h>
+
+#include<impl.h>
+
+int main(int argc, char **argv) {
+    char *ten = malloc(10);
+    do_nasty(ten);
+    free(ten);
+    if(getenv("TEST_ENV")) {
+        printf("TEST_ENV is set.\n");
+    }
+    return 0;
+}

test cases/unit/2 testsetups/impl.c

@@ -0,0 +1,5 @@
+/* Write past the end. */
+
+void do_nasty(char *ptr) {
+    ptr[10] = 'n';
+}

test cases/unit/2 testsetups/impl.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void do_nasty(char *ptr);

test cases/unit/2 testsetups/meson.build

@@ -0,0 +1,16 @@
+project('testsetups', 'c')
+
+vg = find_program('valgrind')
+
+# This is only set when running under Valgrind test setup.
+env = environment()
+env.set('TEST_ENV', '1')
+
+add_test_setup('valgrind',
+  exe_wrapper : [vg, '--error-exitcode=1', '--leak-check=full'],
+  timeout_multiplier : 100,
+  env : env)
+
+buggy = executable('buggy', 'buggy.c', 'impl.c')
+test('Test buggy', buggy)
+