Report expected failures and unexpected passes separately

This makes it clear in the results that tests marked "should_fail"
exist. We also avoid the all caps output and make the classifications
unambigous compared to pytest or autotools' XFAIL/XPASS.

Before:

  OK:       329
  FAIL:       1
  SKIP:       0
  TIMEOUT:    0

After:

  Ok:                 323
  Expected Fail:        1
  Fail:                 6
  Unexpected Pass:      0
  Skipped:              0
  Timeout:              0
pull/4502/head
Dave Reisner 6 years ago committed by Jussi Pakkanen
parent c9e181c10b
commit 96f8842b79
  1. 34
      mesonbuild/mtest.py

@ -144,6 +144,8 @@ class TestResult(enum.Enum):
TIMEOUT = 'TIMEOUT' TIMEOUT = 'TIMEOUT'
SKIP = 'SKIP' SKIP = 'SKIP'
FAIL = 'FAIL' FAIL = 'FAIL'
EXPECTEDFAIL = 'EXPECTEDFAIL'
UNEXPECTEDPASS = 'UNEXPECTEDPASS'
class TestRun: class TestRun:
@ -389,10 +391,10 @@ class SingleTestRunner:
res = TestResult.TIMEOUT res = TestResult.TIMEOUT
elif p.returncode == GNU_SKIP_RETURNCODE: elif p.returncode == GNU_SKIP_RETURNCODE:
res = TestResult.SKIP res = TestResult.SKIP
elif self.test.should_fail == bool(p.returncode): elif self.test.should_fail:
res = TestResult.OK res = TestResult.EXPECTEDFAIL if bool(p.returncode) else TestResult.UNEXPECTEDPASS
else: else:
res = TestResult.FAIL res = TestResult.FAIL if bool(p.returncode) else TestResult.OK
return TestRun(res, p.returncode, self.test.should_fail, duration, stdo, stde, cmd, self.test.env) return TestRun(res, p.returncode, self.test.should_fail, duration, stdo, stde, cmd, self.test.env)
@ -401,6 +403,8 @@ class TestHarness:
self.options = options self.options = options
self.collected_logs = [] self.collected_logs = []
self.fail_count = 0 self.fail_count = 0
self.expectedfail_count = 0
self.unexpectedpass_count = 0
self.success_count = 0 self.success_count = 0
self.skip_count = 0 self.skip_count = 0
self.timeout_count = 0 self.timeout_count = 0
@ -465,6 +469,10 @@ class TestHarness:
self.success_count += 1 self.success_count += 1
elif result.res is TestResult.FAIL: elif result.res is TestResult.FAIL:
self.fail_count += 1 self.fail_count += 1
elif result.res is TestResult.EXPECTEDFAIL:
self.expectedfail_count += 1
elif result.res is TestResult.UNEXPECTEDPASS:
self.unexpectedpass_count += 1
else: else:
sys.exit('Unknown test result encountered: {}'.format(result.res)) sys.exit('Unknown test result encountered: {}'.format(result.res))
@ -480,9 +488,10 @@ class TestHarness:
result_str = '%s %s %s%s%s%5.2f s %s' % \ result_str = '%s %s %s%s%s%5.2f s %s' % \
(num, name, padding1, result.res.value, padding2, result.duration, (num, name, padding1, result.res.value, padding2, result.duration,
status) status)
if not self.options.quiet or result.res is not TestResult.OK: ok_statuses = (TestResult.OK, TestResult.EXPECTEDFAIL)
if result.res is not TestResult.OK and mlog.colorize_console: if not self.options.quiet or result.res not in ok_statuses:
if result.res in (TestResult.FAIL, TestResult.TIMEOUT): if result.res not in ok_statuses and mlog.colorize_console:
if result.res in (TestResult.FAIL, TestResult.TIMEOUT, TestResult.UNEXPECTEDPASS):
decorator = mlog.red decorator = mlog.red
elif result.res is TestResult.SKIP: elif result.res is TestResult.SKIP:
decorator = mlog.yellow decorator = mlog.yellow
@ -503,11 +512,14 @@ class TestHarness:
def print_summary(self): def print_summary(self):
msg = ''' msg = '''
OK: %4d Ok: %4d
FAIL: %4d Expected Fail: %4d
SKIP: %4d Fail: %4d
TIMEOUT: %4d Unexpected Pass: %4d
''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count) Skipped: %4d
Timeout: %4d
''' % (self.success_count, self.expectedfail_count, self.fail_count,
self.unexpectedpass_count, self.skip_count, self.timeout_count)
print(msg) print(msg)
if self.logfile: if self.logfile:
self.logfile.write(msg) self.logfile.write(msg)

Loading…
Cancel
Save