Merge pull request #1575 from centricular/unit-tests-more-backends

Run unit tests on more backends

.appveyor.yml

@@ -4,6 +4,10 @@ os: Visual Studio 2015
 
 environment:
   matrix:
+    - arch: x86
+      compiler: msys2-mingw
+      backend: ninja
+
     - arch: x86
       compiler: msvc2010
       backend: ninja
@@ -20,7 +24,11 @@ environment:
       compiler: msvc2015
       backend: vs2015
 
-    - arch: x86
+    - arch: x64
+      compiler: cygwin
+      backend: ninja
+
+    - arch: x64
       compiler: msys2-mingw
       backend: ninja
 
@@ -34,14 +42,6 @@ environment:
       backend: vs2017
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
 
-    - arch: x64
-      compiler: msys2-mingw
-      backend: ninja
-
-    - arch: x64
-      compiler: cygwin
-      backend: ninja
-
 platform:
   - x64
 

mesonbuild/backend/vs2010backend.py

@@ -418,6 +418,9 @@ class Vs2010Backend(backends.Backend):
                 cmd.append(os.path.join(self.environment.get_build_dir(), self.get_target_filename(i)))
             elif isinstance(i, dependencies.ExternalProgram):
                 cmd += i.get_command()
+            elif isinstance(i, File):
+                relfname = i.rel_to_builddir(self.build_to_src)
+                cmd.append(os.path.join(self.environment.get_build_dir(), relfname))
             else:
                 cmd.append(i)
         cmd_templ = '''"%s" ''' * len(cmd)

run_project_tests.py

@@ -26,6 +26,7 @@ from mesonbuild import mesonlib
 from mesonbuild import mlog
 from mesonbuild import mesonmain
 from mesonbuild.mesonlib import stringlistify, Popen_safe
+from mesonbuild.coredata import backendlist
 import argparse
 import xml.etree.ElementTree as ET
 import time
@@ -33,7 +34,7 @@ import multiprocessing
 import concurrent.futures as conc
 import re
 
-from mesonbuild.coredata import backendlist
+from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
 
 
 class BuildStep(Enum):
@@ -42,6 +43,7 @@ class BuildStep(Enum):
     test = 3
     install = 4
     clean = 5
+    validate = 6
 
 
 class TestResult:
@@ -153,50 +155,36 @@ def stop_handler(signal, frame):
 signal.signal(signal.SIGINT, stop_handler)
 signal.signal(signal.SIGTERM, stop_handler)
 
-# unity_flags = ['--unity']
-unity_flags = []
-
-backend_flags = None
-compile_commands = None
-test_commands = None
-install_commands = []
-clean_commands = []
+# Needed when running cross tests because we don't generate prebuilt files
+compiler = None
 
-def setup_commands(backend):
-    global backend_flags, compile_commands, test_commands, install_commands, clean_commands
+def setup_commands(optbackend):
+    global do_debug, backend, backend_flags
+    global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands
+    backend = optbackend
     msbuild_exe = shutil.which('msbuild')
-    if (backend and backend.startswith('vs')) or (backend is None and msbuild_exe is not None):
-        if backend is None:
-            backend = 'vs2010'
+    # Auto-detect backend if unspecified
+    if backend is None:
+        if msbuild_exe is not None:
+            backend = 'vs' # Meson will auto-detect VS version to use
+        elif mesonlib.is_osx():
+            backend = 'xcode'
+        else:
+            backend = 'ninja'
+    # Set backend arguments for Meson
+    if backend.startswith('vs'):
         backend_flags = ['--backend=' + backend]
-        compile_commands = ['msbuild']
-        test_commands = ['msbuild', 'RUN_TESTS.vcxproj']
-    elif backend == 'xcode' or (backend is None and mesonlib.is_osx()):
+        backend = Backend.vs
+    elif backend == 'xcode':
         backend_flags = ['--backend=xcode']
-        compile_commands = ['xcodebuild']
-        test_commands = ['xcodebuild', '-target', 'RUN_TESTS']
-    else:
-        backend_flags = []
-        # We need at least 1.6 because of -w dupbuild=err
-        ninja_command = environment.detect_ninja(version='1.6')
-        if ninja_command is None:
-            raise RuntimeError('Could not find Ninja v1.6 or newer')
-        if do_debug:
-            compile_commands = [ninja_command, '-v']
-        else:
-            compile_commands = [ninja_command]
-        compile_commands += ['-w', 'dupbuild=err']
-        test_commands = [ninja_command, 'test', 'benchmark']
-        install_commands = [ninja_command, 'install']
-        clean_commands = [ninja_command, 'clean']
-
-def get_compile_commands_for_dir(compile_commands, test_build_dir):
-    if 'msbuild' in compile_commands[0]:
-        sln_name = glob(os.path.join(test_build_dir, '*.sln'))[0]
-        comp = compile_commands + [os.path.split(sln_name)[-1]]
+        backend = Backend.xcode
+    elif backend == 'ninja':
+        backend_flags = ['--backend=ninja']
+        backend = Backend.ninja
     else:
-        comp = compile_commands
-    return comp
+        raise RuntimeError('Unknown backend: {!r}'.format(backend))
+    compile_commands, clean_commands, test_commands, install_commands, \
+        uninstall_commands = get_backend_commands(backend, do_debug)
 
 def get_relative_files_list_from_dir(fromdir):
     paths = []
@@ -223,7 +211,7 @@ def platform_fix_name(fname):
 
     return fname
 
-def validate_install(srcdir, installdir):
+def validate_install(srcdir, installdir, compiler):
     # List of installed files
     info_file = os.path.join(srcdir, 'installed_files.txt')
     # If this exists, the test does not install any other files
@@ -317,18 +305,18 @@ def parse_test_args(testdir):
         pass
     return args
 
-def run_test(skipped, testdir, extra_args, flags, compile_commands, should_fail):
+def run_test(skipped, testdir, extra_args, compiler, backend, flags, commands, should_fail):
     if skipped:
         return None
     with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
         with AutoDeletedDir(tempfile.mkdtemp(prefix='i ', dir=os.getcwd())) as install_dir:
             try:
-                return _run_test(testdir, build_dir, install_dir, extra_args, flags, compile_commands, should_fail)
+                return _run_test(testdir, build_dir, install_dir, extra_args, compiler, backend, flags, commands, should_fail)
             finally:
                 mlog.shutdown() # Close the log file because otherwise Windows wets itself.
 
-def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_commands, should_fail):
-    global install_commands, clean_commands
+def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backend, flags, commands, should_fail):
+    compile_commands, clean_commands, install_commands, uninstall_commands = commands
     test_args = parse_test_args(testdir)
     gen_start = time.time()
     # Configure in-process
@@ -349,9 +337,9 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_c
     if returncode != 0:
         return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     # Build with subprocess
-    comp = get_compile_commands_for_dir(compile_commands, test_build_dir)
+    dir_args = get_backend_args_for_dir(backend, test_build_dir)
     build_start = time.time()
-    pc, o, e = Popen_safe(comp, cwd=test_build_dir)
+    pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
     build_time = time.time() - build_start
     stdo += o
     stde += e
@@ -378,25 +366,26 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_c
         return TestResult('Test that should have failed to run unit tests succeeded', BuildStep.test, stdo, stde, mesonlog, gen_time)
     if returncode != 0:
         return TestResult('Running unit tests failed.', BuildStep.test, stdo, stde, mesonlog, gen_time, build_time, test_time)
-    if len(install_commands) == 0:
-        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time, build_time, test_time)
-    env = os.environ.copy()
-    env['DESTDIR'] = install_dir
-    # Install with subprocess
-    pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
-    stdo += o
-    stde += e
-    if pi.returncode != 0:
-        return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, gen_time, build_time, test_time)
-    if len(clean_commands) != 0:
+    # Do installation, if the backend supports it
+    if len(install_commands) != 0:
         env = os.environ.copy()
-        # Clean with subprocess
-        pi, o, e = Popen_safe(clean_commands, cwd=test_build_dir, env=env)
+        env['DESTDIR'] = install_dir
+        # Install with subprocess
+        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
         stdo += o
         stde += e
         if pi.returncode != 0:
-            return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
-    return TestResult(validate_install(testdir, install_dir), BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
+            return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, gen_time, build_time, test_time)
+    # Clean with subprocess
+    env = os.environ.copy()
+    pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env)
+    stdo += o
+    stde += e
+    if pi.returncode != 0:
+        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
+    if len(install_commands) == 0:
+        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time, build_time, test_time)
+    return TestResult(validate_install(testdir, install_dir, compiler), BuildStep.validate, stdo, stde, mesonlog, gen_time, build_time, test_time)
 
 def gather_tests(testdir):
     tests = [t.replace('\\', '/').split('/', 2)[2] for t in glob(os.path.join(testdir, '*'))]
@@ -421,23 +410,6 @@ def have_java():
         return True
     return False
 
-def using_backend(backends):
-    if isinstance(backends, str):
-        backends = (backends,)
-    for backend in backends:
-        if backend == 'ninja':
-            if not backend_flags:
-                return True
-        elif backend == 'xcode':
-            if backend_flags == '--backend=xcode':
-                return True
-        elif backend == 'vs':
-            if backend_flags.startswith('--backend=vs'):
-                return True
-        else:
-            raise AssertionError('Unknown backend type: ' + backend)
-    return False
-
 def detect_tests_to_run():
     all_tests = []
     all_tests.append(('common', gather_tests('test cases/common'), False))
@@ -450,15 +422,15 @@ def detect_tests_to_run():
     all_tests.append(('platform-windows', gather_tests('test cases/windows'), False if mesonlib.is_windows() or mesonlib.is_cygwin() else True))
     all_tests.append(('platform-linux', gather_tests('test cases/linuxlike'), False if not (mesonlib.is_osx() or mesonlib.is_windows()) else True))
     all_tests.append(('framework', gather_tests('test cases/frameworks'), False if not mesonlib.is_osx() and not mesonlib.is_windows() and not mesonlib.is_cygwin() else True))
-    all_tests.append(('java', gather_tests('test cases/java'), False if using_backend('ninja') and not mesonlib.is_osx() and have_java() else True))
-    all_tests.append(('C#', gather_tests('test cases/csharp'), False if using_backend('ninja') and shutil.which('mcs') else True))
-    all_tests.append(('vala', gather_tests('test cases/vala'), False if using_backend('ninja') and shutil.which('valac') else True))
-    all_tests.append(('rust', gather_tests('test cases/rust'), False if using_backend('ninja') and shutil.which('rustc') else True))
-    all_tests.append(('d', gather_tests('test cases/d'), False if using_backend('ninja') and have_d_compiler() else True))
-    all_tests.append(('objective c', gather_tests('test cases/objc'), False if using_backend(('ninja', 'xcode')) and not mesonlib.is_windows() else True))
-    all_tests.append(('fortran', gather_tests('test cases/fortran'), False if using_backend('ninja') and shutil.which('gfortran') else True))
-    all_tests.append(('swift', gather_tests('test cases/swift'), False if using_backend(('ninja', 'xcode')) and shutil.which('swiftc') else True))
-    all_tests.append(('python3', gather_tests('test cases/python3'), False if using_backend('ninja') and shutil.which('python3') else True))
+    all_tests.append(('java', gather_tests('test cases/java'), False if backend is Backend.ninja and not mesonlib.is_osx() and have_java() else True))
+    all_tests.append(('C#', gather_tests('test cases/csharp'), False if backend is Backend.ninja and shutil.which('mcs') else True))
+    all_tests.append(('vala', gather_tests('test cases/vala'), False if backend is Backend.ninja and shutil.which('valac') else True))
+    all_tests.append(('rust', gather_tests('test cases/rust'), False if backend is Backend.ninja and shutil.which('rustc') else True))
+    all_tests.append(('d', gather_tests('test cases/d'), False if backend is Backend.ninja and have_d_compiler() else True))
+    all_tests.append(('objective c', gather_tests('test cases/objc'), False if backend in (Backend.ninja, Backend.xcode) and not mesonlib.is_windows() else True))
+    all_tests.append(('fortran', gather_tests('test cases/fortran'), False if backend is Backend.ninja and shutil.which('gfortran') else True))
+    all_tests.append(('swift', gather_tests('test cases/swift'), False if backend in (Backend.ninja, Backend.xcode) and shutil.which('swiftc') else True))
+    all_tests.append(('python3', gather_tests('test cases/python3'), False if backend is Backend.ninja and shutil.which('python3') else True))
     return all_tests
 
 def run_tests(all_tests, log_name_base, extra_args):
@@ -473,6 +445,7 @@ def run_tests(all_tests, log_name_base, extra_args):
     passing_tests = 0
     failing_tests = 0
     skipped_tests = 0
+    commands = (compile_commands, clean_commands, install_commands, uninstall_commands)
 
     try:
         # This fails in some CI environments for unknown reasons.
@@ -502,7 +475,7 @@ def run_tests(all_tests, log_name_base, extra_args):
             should_fail = False
             if name.startswith('failing'):
                 should_fail = name.split('failing-')[1]
-            result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, should_fail)
+            result = executor.submit(run_test, skipped, t, extra_args, compiler, backend, backend_flags, commands, should_fail)
             futures.append((testname, t, result))
         for (testname, t, result) in futures:
             sys.stdout.flush()
@@ -617,7 +590,7 @@ def generate_prebuilt():
     return objectfile, stlibfile
 
 def check_meson_commands_work():
-    global meson_command, compile_commands, test_commands, install_commands
+    global backend, meson_command, compile_commands, test_commands, install_commands
     testdir = 'test cases/common/1 trivial'
     with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
         print('Checking that configuring works...')
@@ -626,8 +599,8 @@ def check_meson_commands_work():
         if pc.returncode != 0:
             raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(testdir, e, o))
         print('Checking that building works...')
-        compile_cmd = get_compile_commands_for_dir(compile_commands, build_dir)
-        pc, o, e = Popen_safe(compile_cmd, cwd=build_dir)
+        dir_args = get_backend_args_for_dir(backend, build_dir)
+        pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir)
         if pc.returncode != 0:
             raise RuntimeError('Failed to build {!r}:\n{}\n{}'.format(testdir, e, o))
         print('Checking that testing works...')
@@ -649,21 +622,6 @@ if __name__ == '__main__':
     options = parser.parse_args()
     setup_commands(options.backend)
 
-    # Appveyor sets the `platform` environment variable which completely messes
-    # up building with the vs2010 and vs2015 backends.
-    #
-    # Specifically, MSBuild reads the `platform` environment variable to set
-    # the configured value for the platform (Win32/x64/arm), which breaks x86
-    # builds.
-    #
-    # Appveyor setting this also breaks our 'native build arch' detection for
-    # Windows in environment.py:detect_windows_arch() by overwriting the value
-    # of `platform` set by vcvarsall.bat.
-    #
-    # While building for x86, `platform` should be unset.
-    if 'APPVEYOR' in os.environ and os.environ['arch'] == 'x86':
-        os.environ.pop('platform')
-
     script_dir = os.path.split(__file__)[0]
     if script_dir != '':
         os.chdir(script_dir)

run_tests.py

@@ -20,28 +20,138 @@ import shutil
 import subprocess
 import platform
 from mesonbuild import mesonlib
+from mesonbuild.environment import detect_ninja
+from enum import Enum
+from glob import glob
 
-def using_vs_backend():
-    for arg in sys.argv[1:]:
-        if arg.startswith('--backend=vs'):
-            return True
-    return False
+Backend = Enum('Backend', 'ninja vs xcode')
+
+if mesonlib.is_windows() or mesonlib.is_cygwin():
+    exe_suffix = '.exe'
+else:
+    exe_suffix = ''
+
+def get_backend_args_for_dir(backend, builddir):
+    '''
+    Visual Studio backend needs to be given the solution to build
+    '''
+    if backend is Backend.vs:
+        sln_name = glob(os.path.join(builddir, '*.sln'))[0]
+        return [os.path.split(sln_name)[-1]]
+    return []
+
+def find_vcxproj_with_target(builddir, target):
+    import re, fnmatch
+    t, ext = os.path.splitext(target)
+    if ext:
+        p = '<TargetName>{}</TargetName>\s*<TargetExt>\{}</TargetExt>'.format(t, ext)
+    else:
+        p = '<TargetName>{}</TargetName>'.format(t)
+    for root, dirs, files in os.walk(builddir):
+        for f in fnmatch.filter(files, '*.vcxproj'):
+            f = os.path.join(builddir, f)
+            with open(f, 'r', encoding='utf-8') as o:
+                if re.search(p, o.read(), flags=re.MULTILINE):
+                    return f
+    raise RuntimeError('No vcxproj matching {!r} in {!r}'.format(p, builddir))
+
+def get_builddir_target_args(backend, builddir, target):
+    dir_args = []
+    if not target:
+        dir_args = get_backend_args_for_dir(backend, builddir)
+    if target is None:
+        return dir_args
+    if backend is Backend.vs:
+        vcxproj = find_vcxproj_with_target(builddir, target)
+        target_args = [vcxproj]
+    elif backend is Backend.xcode:
+        target_args = ['-target', target]
+    elif backend is Backend.ninja:
+        target_args = [target]
+    else:
+        raise AssertionError('Unknown backend: {!r}'.format(backend))
+    return target_args + dir_args
+
+def get_backend_commands(backend, debug=False):
+    install_cmd = []
+    uninstall_cmd = []
+    if backend is Backend.vs:
+        cmd = ['msbuild']
+        clean_cmd = cmd + ['/target:Clean']
+        test_cmd = cmd + ['RUN_TESTS.vcxproj']
+    elif backend is Backend.xcode:
+        cmd = ['xcodebuild']
+        clean_cmd = cmd + ['-alltargets', 'clean']
+        test_cmd = cmd + ['-target', 'RUN_TESTS']
+    elif backend is Backend.ninja:
+        # We need at least 1.6 because of -w dupbuild=err
+        cmd = [detect_ninja('1.6'), '-w', 'dupbuild=err']
+        if cmd[0] is None:
+            raise RuntimeError('Could not find Ninja v1.6 or newer')
+        if debug:
+            cmd += ['-v']
+        clean_cmd = cmd + ['clean']
+        test_cmd = cmd + ['test', 'benchmark']
+        install_cmd = cmd + ['install']
+        uninstall_cmd = cmd + ['uninstall']
+    else:
+        raise AssertionError('Unknown backend: {!r}'.format(backend))
+    return cmd, clean_cmd, test_cmd, install_cmd, uninstall_cmd
+
+def get_fake_options(prefix):
+    import argparse
+    opts = argparse.Namespace()
+    opts.cross_file = None
+    opts.wrap_mode = None
+    opts.prefix = prefix
+    return opts
+
+class FakeEnvironment(object):
+    def __init__(self):
+        self.cross_info = None
+
+    def is_cross_build(self):
+        return False
 
 if __name__ == '__main__':
     returncode = 0
+    # Iterate over list in reverse order to find the last --backend arg
+    backend = Backend.ninja
+    for arg in reversed(sys.argv[1:]):
+        if arg.startswith('--backend'):
+            if arg.startswith('--backend=vs'):
+                backend = Backend.vs
+            elif arg == '--backend=xcode':
+                backend = Backend.xcode
+            break
     # Running on a developer machine? Be nice!
     if not mesonlib.is_windows() and 'TRAVIS' not in os.environ:
         os.nice(20)
+    # Appveyor sets the `platform` environment variable which completely messes
+    # up building with the vs2010 and vs2015 backends.
+    #
+    # Specifically, MSBuild reads the `platform` environment variable to set
+    # the configured value for the platform (Win32/x64/arm), which breaks x86
+    # builds.
+    #
+    # Appveyor setting this also breaks our 'native build arch' detection for
+    # Windows in environment.py:detect_windows_arch() by overwriting the value
+    # of `platform` set by vcvarsall.bat.
+    #
+    # While building for x86, `platform` should be unset.
+    if 'APPVEYOR' in os.environ and os.environ['arch'] == 'x86':
+        os.environ.pop('platform')
+    # Run tests
     print('Running unittests.\n')
     units = ['InternalTests', 'AllPlatformTests']
     if mesonlib.is_linux():
         units += ['LinuxlikeTests']
     elif mesonlib.is_windows():
         units += ['WindowsTests']
-    # Unit tests always use the Ninja backend, so just skip them if we're
-    # testing the VS backend
-    if not using_vs_backend():
-        returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units)
+    # Can't pass arguments to unit tests, so set the backend to use in the environment
+    env = os.environ.copy()
+    env['MESON_UNIT_TEST_BACKEND'] = backend.name
+    returncode += subprocess.call([sys.executable, 'run_unittests.py', '-v'] + units, env=env)
     # Ubuntu packages do not have a binary without -6 suffix.
     if shutil.which('arm-linux-gnueabihf-gcc-6') and not platform.machine().startswith('arm'):
         print('Running cross compilation tests.\n')

run_unittests.py

@@ -25,13 +25,12 @@ import mesonbuild.compilers
 import mesonbuild.environment
 import mesonbuild.mesonlib
 from mesonbuild.mesonlib import is_windows, is_osx, is_cygwin
-from mesonbuild.environment import detect_ninja, Environment
+from mesonbuild.environment import Environment
 from mesonbuild.dependencies import PkgConfigDependency, ExternalProgram
 
-if is_windows() or is_cygwin():
-    exe_suffix = '.exe'
-else:
-    exe_suffix = ''
+from run_tests import exe_suffix, get_fake_options, FakeEnvironment
+from run_tests import get_builddir_target_args, get_backend_commands, Backend
+
 
 def get_soname(fname):
     # HACK, fix to not use shell.
@@ -44,21 +43,6 @@ def get_soname(fname):
             return m.group(1)
     raise RuntimeError('Could not determine soname:\n\n' + raw_out)
 
-def get_fake_options(prefix):
-    import argparse
-    opts = argparse.Namespace()
-    opts.cross_file = None
-    opts.wrap_mode = None
-    opts.prefix = prefix
-    return opts
-
-class FakeEnvironment(object):
-    def __init__(self):
-        self.cross_info = None
-
-    def is_cross_build(self):
-        return False
-
 class InternalTests(unittest.TestCase):
 
     def test_version_number(self):
@@ -346,11 +330,18 @@ class BasePlatformTests(unittest.TestCase):
         self.prefix = '/usr'
         self.libdir = os.path.join(self.prefix, 'lib')
         self.installdir = os.path.join(self.builddir, 'install')
-        self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py')]
+        # Get the backend
+        # FIXME: Extract this from argv?
+        self.backend = getattr(Backend, os.environ.get('MESON_UNIT_TEST_BACKEND', 'ninja'))
+        self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py'),
+                              '--backend=' + self.backend.name]
         self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')]
         self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')]
         self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir]
-        self.ninja_command = [detect_ninja(), '-C', self.builddir]
+        # Backend-specific build commands
+        self.build_command, self.clean_command, self.test_command, self.install_command, \
+            self.uninstall_command = get_backend_commands(self.backend)
+        # Test directories
         self.common_test_dir = os.path.join(src_root, 'test cases/common')
         self.vala_test_dir = os.path.join(src_root, 'test cases/vala')
         self.framework_test_dir = os.path.join(src_root, 'test cases/frameworks')
@@ -370,14 +361,14 @@ class BasePlatformTests(unittest.TestCase):
         os.environ = self.orig_env
         super().tearDown()
 
-    def _run(self, command):
+    def _run(self, command, workdir=None):
         '''
         Run a command while printing the stdout and stderr to stdout,
         and also return a copy of it
         '''
         p = subprocess.Popen(command, stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT, env=os.environ.copy(),
-                             universal_newlines=True)
+                             universal_newlines=True, cwd=workdir)
         output = p.communicate()[0]
         print(output)
         if p.returncode != 0:
@@ -398,27 +389,36 @@ class BasePlatformTests(unittest.TestCase):
             raise
         self.privatedir = os.path.join(self.builddir, 'meson-private')
 
-    def build(self, extra_args=None):
+    def build(self, target=None, extra_args=None):
         if extra_args is None:
             extra_args = []
-        self._run(self.ninja_command + extra_args)
+        # Add arguments for building the target (if specified),
+        # and using the build dir (if required, with VS)
+        args = get_builddir_target_args(self.backend, self.builddir, target)
+        self._run(self.build_command + args + extra_args, workdir=self.builddir)
+
+    def clean(self):
+        dir_args = get_builddir_target_args(self.backend, self.builddir, None)
+        self._run(self.clean_command + dir_args, workdir=self.builddir)
 
     def run_tests(self):
-        self._run(self.ninja_command + ['test'])
+        self._run(self.test_command, workdir=self.builddir)
 
     def install(self):
+        if self.backend is not Backend.ninja:
+            raise unittest.SkipTest('{!r} backend can\'t install files'.format(self.backend.name))
         os.environ['DESTDIR'] = self.installdir
-        self._run(self.ninja_command + ['install'])
+        self._run(self.install_command, workdir=self.builddir)
 
     def uninstall(self):
-        self._run(self.ninja_command + ['uninstall'])
+        self._run(self.uninstall_command, workdir=self.builddir)
 
     def run_target(self, target):
         '''
         Run a Ninja target while printing the stdout and stderr to stdout,
         and also return a copy of it
         '''
-        return self._run(self.ninja_command + [target])
+        return self.build(target=target)
 
     def setconf(self, arg, will_build=True):
         # This is needed to increase the difference between build.ninja's
@@ -432,13 +432,15 @@ class BasePlatformTests(unittest.TestCase):
         shutil.rmtree(self.builddir)
 
     def get_compdb(self):
+        if self.backend is not Backend.ninja:
+            raise unittest.SkipTest('Compiler db not available with {} backend'.format(self.backend.name))
         with open(os.path.join(self.builddir, 'compile_commands.json')) as ifile:
             contents = json.load(ifile)
         # If Ninja is using .rsp files, generate them, read their contents, and
         # replace it as the command for all compile commands in the parsed json.
         if len(contents) > 0 and contents[0]['command'].endswith('.rsp'):
             # Pretend to build so that the rsp files are generated
-            self.build(['-d', 'keeprsp', '-n'])
+            self.build(extra_args=['-d', 'keeprsp', '-n'])
             for each in contents:
                 # Extract the actual command from the rsp file
                 compiler, rsp = each['command'].split(' @')
@@ -613,6 +615,8 @@ class AllPlatformTests(BasePlatformTests):
         Tests that the Meson introspection API exposes install filenames correctly
         https://github.com/mesonbuild/meson/issues/829
         '''
+        if self.backend is not Backend.ninja:
+            raise unittest.SkipTest('{!r} backend can\'t install files'.format(self.backend.name))
         testdir = os.path.join(self.common_test_dir, '8 install')
         self.init(testdir)
         intro = self.introspect('--targets')
@@ -722,7 +726,7 @@ class AllPlatformTests(BasePlatformTests):
         exe = os.path.join(self.builddir, 'fooprog' + exe_suffix)
         self.assertTrue(os.path.exists(genfile))
         self.assertFalse(os.path.exists(exe))
-        self._run(self.ninja_command + ['fooprog' + exe_suffix])
+        self.build(target=('fooprog' + exe_suffix))
         self.assertTrue(os.path.exists(exe))
 
     def test_internal_include_order(self):
@@ -964,6 +968,15 @@ class AllPlatformTests(BasePlatformTests):
         os.environ['CFLAGS'] = '-DMESON_FAIL_VALUE=cflags-read'.format(define)
         self.init(testdir, ['-D{}={}'.format(define, value)])
 
+    def test_custom_target_exe_data_deterministic(self):
+        testdir = os.path.join(self.common_test_dir, '117 custom target capture')
+        self.init(testdir)
+        meson_exe_dat1 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
+        self.wipe()
+        self.init(testdir)
+        meson_exe_dat2 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
+        self.assertListEqual(meson_exe_dat1, meson_exe_dat2)
+
 
 class WindowsTests(BasePlatformTests):
     '''
@@ -1231,15 +1244,6 @@ class LinuxlikeTests(BasePlatformTests):
             Oargs = [arg for arg in cmd if arg.startswith('-O')]
             self.assertEqual(Oargs, [Oflag, '-O0'])
 
-    def test_custom_target_exe_data_deterministic(self):
-        testdir = os.path.join(self.common_test_dir, '117 custom target capture')
-        self.init(testdir)
-        meson_exe_dat1 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
-        self.wipe()
-        self.init(testdir)
-        meson_exe_dat2 = glob(os.path.join(self.privatedir, 'meson_exe*.dat'))
-        self.assertListEqual(meson_exe_dat1, meson_exe_dat2)
-
     def _test_stds_impl(self, testdir, compiler, p):
         lang_std = p + '_std'
         # Check that all the listed -std=xxx options for this compiler work