Report xpass results as failures.

docs/markdown/snippets/xfail.md

@@ -0,0 +1,15 @@
+## Tests that should fail but did not are now errors
+
+You can tag a test as needing to fail like this:
+
+```meson
+test('shoulfail', exe, should_fail: true)
+```
+
+If the test passes the problem is reported in the error logs but due
+to a bug it was not reported in the test runner's exit code. Starting
+from this release the unexpected passes are properly reported in the
+test runner's exit code. This means that test runs that were passing
+in earlier versions of Meson will report failures with the current
+version. This is a good thing, though, since it reveals an error in
+your test suite that has, until now, gone unnoticed.

mesonbuild/mtest.py

@@ -664,7 +664,6 @@ class TestHarness:
     def process_test_result(self, result):
         if result.res is TestResult.TIMEOUT:
             self.timeout_count += 1
-            self.fail_count += 1
         elif result.res is TestResult.SKIP:
             self.skip_count += 1
         elif result.res is TestResult.OK:
@@ -746,6 +745,9 @@ Timeout:            %4d
                         line = line.encode('ascii', errors='replace').decode()
                         print(line)
 
+    def total_failure_count(self):
+        return self.fail_count + self.unexpectedpass_count + self.timeout_count
+
     def doit(self):
         if self.is_run:
             raise RuntimeError('Test harness object can only be used once.')
@@ -754,7 +756,7 @@ Timeout:            %4d
         if not tests:
             return 0
         self.run_tests(tests)
-        return self.fail_count
+        return self.total_failure_count()
 
     @staticmethod
     def split_suite_string(suite):
@@ -939,7 +941,7 @@ Timeout:            %4d
         if not tests:
             return 0
         self.run_tests(tests)
-        return self.fail_count
+        return self.total_failure_count()
 
 
 def list_tests(th):

test cases/failing test/6 xpass/meson.build

@@ -0,0 +1,4 @@
+project('unexpected pass', 'c')
+
+test('should_fail_but_does_not', executable('xpass', 'xpass.c'),
+     should_fail: true)

test cases/failing test/6 xpass/xpass.c

@@ -0,0 +1 @@
+int main(int argc, char **argv) { return 0; }