Report expected failures and unexpected passes separately

This makes it clear in the results that tests marked "should_fail"
exist. We also avoid the all caps output and make the classifications
unambigous compared to pytest or autotools' XFAIL/XPASS.

Before:

  OK:       329
  FAIL:       1
  SKIP:       0
  TIMEOUT:    0

After:

  Ok:                 323
  Expected Fail:        1
  Fail:                 6
  Unexpected Pass:      0
  Skipped:              0
  Timeout:              0
pull/4502/head
Dave Reisner 6 years ago committed by Jussi Pakkanen
parent c9e181c10b
commit 96f8842b79
  1. 34
      mesonbuild/mtest.py

@ -144,6 +144,8 @@ class TestResult(enum.Enum):
TIMEOUT = 'TIMEOUT'
SKIP = 'SKIP'
FAIL = 'FAIL'
EXPECTEDFAIL = 'EXPECTEDFAIL'
UNEXPECTEDPASS = 'UNEXPECTEDPASS'
class TestRun:
@ -389,10 +391,10 @@ class SingleTestRunner:
res = TestResult.TIMEOUT
elif p.returncode == GNU_SKIP_RETURNCODE:
res = TestResult.SKIP
elif self.test.should_fail == bool(p.returncode):
res = TestResult.OK
elif self.test.should_fail:
res = TestResult.EXPECTEDFAIL if bool(p.returncode) else TestResult.UNEXPECTEDPASS
else:
res = TestResult.FAIL
res = TestResult.FAIL if bool(p.returncode) else TestResult.OK
return TestRun(res, p.returncode, self.test.should_fail, duration, stdo, stde, cmd, self.test.env)
@ -401,6 +403,8 @@ class TestHarness:
self.options = options
self.collected_logs = []
self.fail_count = 0
self.expectedfail_count = 0
self.unexpectedpass_count = 0
self.success_count = 0
self.skip_count = 0
self.timeout_count = 0
@ -465,6 +469,10 @@ class TestHarness:
self.success_count += 1
elif result.res is TestResult.FAIL:
self.fail_count += 1
elif result.res is TestResult.EXPECTEDFAIL:
self.expectedfail_count += 1
elif result.res is TestResult.UNEXPECTEDPASS:
self.unexpectedpass_count += 1
else:
sys.exit('Unknown test result encountered: {}'.format(result.res))
@ -480,9 +488,10 @@ class TestHarness:
result_str = '%s %s %s%s%s%5.2f s %s' % \
(num, name, padding1, result.res.value, padding2, result.duration,
status)
if not self.options.quiet or result.res is not TestResult.OK:
if result.res is not TestResult.OK and mlog.colorize_console:
if result.res in (TestResult.FAIL, TestResult.TIMEOUT):
ok_statuses = (TestResult.OK, TestResult.EXPECTEDFAIL)
if not self.options.quiet or result.res not in ok_statuses:
if result.res not in ok_statuses and mlog.colorize_console:
if result.res in (TestResult.FAIL, TestResult.TIMEOUT, TestResult.UNEXPECTEDPASS):
decorator = mlog.red
elif result.res is TestResult.SKIP:
decorator = mlog.yellow
@ -503,11 +512,14 @@ class TestHarness:
def print_summary(self):
msg = '''
OK: %4d
FAIL: %4d
SKIP: %4d
TIMEOUT: %4d
''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count)
Ok: %4d
Expected Fail: %4d
Fail: %4d
Unexpected Pass: %4d
Skipped: %4d
Timeout: %4d
''' % (self.success_count, self.expectedfail_count, self.fail_count,
self.unexpectedpass_count, self.skip_count, self.timeout_count)
print(msg)
if self.logfile:
self.logfile.write(msg)

Loading…
Cancel
Save