Created new suite of tests that should fail when executed.

run_tests.py

@@ -30,7 +30,7 @@ if True: # Currently we have only one backend.
     test_commands = [ninja_command, 'test']
     install_commands = [ninja_command, 'install']
 
-def run_test(testdir):
+def run_test(testdir, should_succeed=True):
     shutil.rmtree(test_build_dir)
     shutil.rmtree(install_dir)
     os.mkdir(test_build_dir)
@@ -39,6 +39,10 @@ def run_test(testdir):
     gen_command = [sys.executable, meson_command, '--prefix', install_dir, testdir, test_build_dir] + backend_flags
     p = subprocess.Popen(gen_command)
     p.wait()
+    if not should_succeed:
+        if p.returncode != 0:
+            return
+        raise RuntimeError('Test that should fail succeeded.')
     if p.returncode != 0:
         raise RuntimeError('Generating the build system failed.')
     pc = subprocess.Popen(compile_commands, cwd=test_build_dir)
@@ -63,6 +67,7 @@ def gather_tests(testdir):
 
 def run_tests():
     commontests = gather_tests('test cases/common')
+    failtests = gather_tests('test cases/failing')
     if environment.is_osx():
         platformtests = gather_tests('test cases/osx')
     elif environment.is_windows():
@@ -87,12 +92,23 @@ def run_tests():
         pass
     print('\nRunning common tests.\n')
     [run_test(t) for t in commontests]
-    print('\nRunning platform dependent tests.\n')
-    [run_test(t) for t in platformtests]
-    print('\nRunning framework tests.\n')
-    [run_test(t) for t in frameworktests]
-    print('\nRunning extra language tests.\n')
-    [run_test(t) for t in objctests]
+    print('\nRunning failing tests.\n')
+    [run_test(t, False) for t in failtests]
+    if len(platformtests) > 0:
+        print('\nRunning platform dependent tests.\n')
+        [run_test(t) for t in platformtests]
+    else:
+        print('\nNo platform specific tests.\n')
+    if len(frameworktests) > 0:
+        print('\nRunning framework tests.\n')
+        [run_test(t) for t in frameworktests]
+    else:
+        print('\nNo framework tests on this platform.\n')
+    if len(objctests) > 0:
+        print('\nRunning extra language tests.\n')
+        [run_test(t) for t in objctests]
+    else:
+        print('\nNo extra language tests on this platform.\n')
 
 def check_file(fname):
     if fname.endswith('parsetab.py'): # Autogenerated

test cases/failing/1 project not first/meson.build

@@ -0,0 +1,4 @@
+var = 'assignment before project() call'
+project('no worky', 'c')
+
+test('not run', executable('prog', 'prog.c'))

test cases/failing/1 project not first/prog.c

@@ -0,0 +1 @@
+int main(int argc, char **argv) { return 0; }