From 2a64ed855e6db81c3df4659e1066f777b9bacc8c Mon Sep 17 00:00:00 2001
From: Jon Turney <jon.turney@dronecode.org.uk>
Date: Fri, 27 Oct 2017 14:39:10 +0100
Subject: [PATCH] Verify that failing tests are failing with an error, not a
 python exception

PR #2527 suggests "making failing tests more strict about failing
gracefully".

To achive this, make meson exit with distinct exit statuses for meson errors
and python exceptions, and check the exit status is as expected for failing
tests.

I can't see how to write a test for this, within the current framework.

You can test this change by reverting the fix (but not the test) from commit
1a159db8 and verifying that 'test cases/failing/66 string as link target'
fails.
---
 docs/markdown/Running-Meson.md |  6 ++++++
 man/meson.1                    | 13 +++++++++++++
 mesonbuild/mesonmain.py        |  3 ++-
 run_project_tests.py           |  7 +++++--
 4 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/docs/markdown/Running-Meson.md b/docs/markdown/Running-Meson.md
index 23d5e97b5..14b2d1990 100644
--- a/docs/markdown/Running-Meson.md
+++ b/docs/markdown/Running-Meson.md
@@ -146,3 +146,9 @@ Meson has a standard command line help feature. It can be accessed
 with the following command.
 
     meson --help
+
+Exit status
+==
+
+Meson exits with status 0 if successful, 1 for problems with the command line or
+meson.build file, and 2 for internal errors.
diff --git a/man/meson.1 b/man/meson.1
index 929bc6e32..4429fa2fb 100644
--- a/man/meson.1
+++ b/man/meson.1
@@ -202,6 +202,19 @@ show available versions of the specified project
 \fBstatus\fR
 show installed and available versions of currently used subprojects
 
+.SH EXIT STATUS
+
+.TP
+.B 0
+Successful.
+.TP
+.B 1
+Usage error, or an error parsing or executing meson.build.
+.TP
+.B 2
+Internal error.
+.TP
+
 .SH SEE ALSO
 
 http://mesonbuild.com/
diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py
index 073e5052f..7966d7095 100644
--- a/mesonbuild/mesonmain.py
+++ b/mesonbuild/mesonmain.py
@@ -376,11 +376,12 @@ def run(original_args, mainfile=None):
             mlog.log("\nA full log can be found at", mlog.bold(logfile))
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
+            return 1
         else:
             if os.environ.get('MESON_FORCE_BACKTRACE'):
                 raise
             traceback.print_exc()
-        return 1
+            return 2
     finally:
         mlog.shutdown()
 
diff --git a/run_project_tests.py b/run_project_tests.py
index 1d17000b5..9bbbbd093 100755
--- a/run_project_tests.py
+++ b/run_project_tests.py
@@ -321,9 +321,12 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backen
         mesonlog = no_meson_log_msg
     gen_time = time.time() - gen_start
     if should_fail == 'meson':
-        if returncode != 0:
+        if returncode == 1:
             return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
-        return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        elif returncode != 0:
+            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time)
+        else:
+            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     if returncode != 0:
         return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
     # Touch the meson.build file to force a regenerate so we can test that