unit tests: Add class to generate failing tests

It is not feasible to test all failure modes by creating projects in
`test cases/failing` that would be an explosion of files, and that
mechanism is too coarse anyway. We have no way to ensure that the
expected error is being raised.

See FailureTests.test_dependency for an example.

mesonbuild/dependencies/base.py

@@ -59,7 +59,10 @@ class Dependency:
         self.compile_args = []
         self.link_args = []
         self.sources = []
-        method = DependencyMethods(kwargs.get('method', 'auto'))
+        method = kwargs.get('method', 'auto')
+        if method not in [e.value for e in DependencyMethods]:
+            raise DependencyException('method {!r} is invalid'.format(method))
+        method = DependencyMethods(method)
 
         # Set the detection method. If the method is set to auto, use any available method.
         # If method is set to a specific string, allow only that detection method.
@@ -68,7 +71,7 @@ class Dependency:
         elif method in self.get_methods():
             self.methods = [method]
         else:
-            raise MesonException(
+            raise DependencyException(
                 'Unsupported detection method: {}, allowed methods are {}'.format(
                     method.value,
                     mlog.format_list(map(lambda x: x.value, [DependencyMethods.AUTO] + self.get_methods()))))

run_project_tests.py

@@ -34,7 +34,7 @@ import time
 import multiprocessing
 import concurrent.futures as conc
 import re
-from run_unittests import get_fake_options
+from run_unittests import get_fake_options, run_configure_inprocess
 
 from run_tests import get_backend_commands, get_backend_args_for_dir, Backend
 from run_tests import ensure_backend_detects_changes
@@ -249,18 +249,6 @@ def log_text_file(logfile, testdir, stdo, stde):
         executor.shutdown()
         raise StopException()
 
-def run_configure_inprocess(commandlist):
-    old_stdout = sys.stdout
-    sys.stdout = mystdout = StringIO()
-    old_stderr = sys.stderr
-    sys.stderr = mystderr = StringIO()
-    try:
-        returncode = mesonmain.run(commandlist[0], commandlist[1:])
-    finally:
-        sys.stdout = old_stdout
-        sys.stderr = old_stderr
-    return returncode, mystdout.getvalue(), mystderr.getvalue()
-
 def run_test_inprocess(testdir):
     old_stdout = sys.stdout
     sys.stdout = mystdout = StringIO()

run_tests.py

@@ -22,7 +22,9 @@ import subprocess
 import tempfile
 import platform
 from mesonbuild import mesonlib
+from mesonbuild import mesonmain
 from mesonbuild.environment import detect_ninja
+from io import StringIO
 from enum import Enum
 from glob import glob
 
@@ -118,6 +120,18 @@ def get_fake_options(prefix):
 def should_run_linux_cross_tests():
     return shutil.which('arm-linux-gnueabihf-gcc-6') and not platform.machine().startswith('arm')
 
+def run_configure_inprocess(commandlist):
+    old_stdout = sys.stdout
+    sys.stdout = mystdout = StringIO()
+    old_stderr = sys.stderr
+    sys.stderr = mystderr = StringIO()
+    try:
+        returncode = mesonmain.run(commandlist[0], commandlist[1:])
+    finally:
+        sys.stdout = old_stdout
+        sys.stderr = old_stderr
+    return returncode, mystdout.getvalue(), mystderr.getvalue()
+
 class FakeEnvironment(object):
     def __init__(self):
         self.cross_info = None
@@ -164,7 +178,7 @@ if __name__ == '__main__':
         os.environ.pop('platform')
     # Run tests
     print('Running unittests.\n')
-    units = ['InternalTests', 'AllPlatformTests']
+    units = ['InternalTests', 'AllPlatformTests', 'FailureTests']
     if mesonlib.is_linux():
         units += ['LinuxlikeTests']
         if should_run_linux_cross_tests():

run_unittests.py

@@ -25,16 +25,19 @@ import unittest
 from configparser import ConfigParser
 from glob import glob
 from pathlib import PurePath
+
+import mesonbuild.mlog
 import mesonbuild.compilers
 import mesonbuild.environment
 import mesonbuild.mesonlib
 from mesonbuild.mesonlib import is_windows, is_osx, is_cygwin
 from mesonbuild.environment import Environment
+from mesonbuild.dependencies import DependencyException
 from mesonbuild.dependencies import PkgConfigDependency, ExternalProgram
 
 from run_tests import exe_suffix, get_fake_options, FakeEnvironment
 from run_tests import get_builddir_target_args, get_backend_commands, Backend
-from run_tests import ensure_backend_detects_changes
+from run_tests import ensure_backend_detects_changes, run_configure_inprocess
 
 
 def get_dynamic_section_entry(fname, entry):
@@ -401,8 +404,8 @@ class BasePlatformTests(unittest.TestCase):
         # Get the backend
         # FIXME: Extract this from argv?
         self.backend = getattr(Backend, os.environ.get('MESON_UNIT_TEST_BACKEND', 'ninja'))
-        self.meson_command = [sys.executable, os.path.join(src_root, 'meson.py'),
-                              '--backend=' + self.backend.name]
+        self.meson_args = [os.path.join(src_root, 'meson.py'), '--backend=' + self.backend.name]
+        self.meson_command = [sys.executable] + self.meson_args
         self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')]
         self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')]
         self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir]
@@ -452,7 +455,7 @@ class BasePlatformTests(unittest.TestCase):
             raise subprocess.CalledProcessError(p.returncode, command)
         return output
 
-    def init(self, srcdir, extra_args=None, default_args=True):
+    def init(self, srcdir, extra_args=None, default_args=True, inprocess=False):
         self.assertTrue(os.path.exists(srcdir))
         if extra_args is None:
             extra_args = []
@@ -462,14 +465,26 @@ class BasePlatformTests(unittest.TestCase):
         if default_args:
             args += ['--prefix', self.prefix,
                      '--libdir', self.libdir]
-        try:
-            self._run(self.meson_command + args + extra_args)
-        except unittest.SkipTest:
-            raise unittest.SkipTest('Project requested skipping: ' + srcdir)
-        except:
-            self._print_meson_log()
-            raise
         self.privatedir = os.path.join(self.builddir, 'meson-private')
+        if inprocess:
+            try:
+                run_configure_inprocess(self.meson_args + args + extra_args)
+            except:
+                self._print_meson_log()
+                raise
+            finally:
+                # Close log file to satisfy Windows file locking
+                mesonbuild.mlog.shutdown()
+                mesonbuild.mlog.log_dir = None
+                mesonbuild.mlog.log_file = None
+        else:
+            try:
+                self._run(self.meson_command + args + extra_args)
+            except unittest.SkipTest:
+                raise unittest.SkipTest('Project requested skipping: ' + srcdir)
+            except:
+                self._print_meson_log()
+                raise
 
     def build(self, target=None, extra_args=None):
         if extra_args is None:
@@ -1196,6 +1211,52 @@ int main(int argc, char **argv) {
                 self.assertTrue(path.startswith('$ORIGIN'), msg=(each, path))
 
 
+class FailureTests(BasePlatformTests):
+    '''
+    Tests that test failure conditions. Build files here should be dynamically
+    generated and static tests should go into `test cases/failing*`.
+    This is useful because there can be many ways in which a particular
+    function can fail, and creating failing tests for all of them is tedious
+    and slows down testing.
+    '''
+    dnf = "[Dd]ependency.*not found"
+
+    def setUp(self):
+        super().setUp()
+        self.srcdir = os.path.realpath(tempfile.mkdtemp())
+        self.mbuild = os.path.join(self.srcdir, 'meson.build')
+
+    def tearDown(self):
+        super().tearDown()
+        shutil.rmtree(self.srcdir)
+
+    def assertMesonRaises(self, contents, match, extra_args=None):
+        '''
+        Assert that running meson configure on the specified contents raises
+        the specified error message.
+        '''
+        with open(self.mbuild, 'w') as f:
+            f.write("project('failure test', 'c', 'cpp')\n")
+            f.write(contents)
+        # Force tracebacks so we can detect them properly
+        os.environ['MESON_FORCE_BACKTRACE'] = '1'
+        with self.assertRaisesRegex(DependencyException, match, msg=contents):
+            # Must run in-process or we'll get a generic CalledProcessError
+            self.init(self.srcdir, extra_args=extra_args, inprocess=True)
+
+    def test_dependency(self):
+        if not shutil.which('pkg-config'):
+            raise unittest.SkipTest('pkg-config not found')
+        a = (("dependency('zlib', method : 'fail')", "'fail' is invalid"),
+             ("dependency('zlib', static : '1')", "[Ss]tatic.*boolean"),
+             ("dependency('zlib', version : 1)", "[Vv]ersion.*string or list"),
+             ("dependency('zlib', required : 1)", "[Rr]equired.*boolean"),
+             ("dependency('zlib', method : 1)", "[Mm]ethod.*string"),
+             ("dependency('zlibfail')", self.dnf),)
+        for contents, match in a:
+            self.assertMesonRaises(contents, match)
+
+
 class WindowsTests(BasePlatformTests):
     '''
     Tests that should run on Cygwin, MinGW, and MSVC