mtest: warn on invalid TAP output

In commit a7e458effa we stopped erroring
out on invalid TAP stream contents, with the rationale that "prove" has
become more lenient.

A close reading of the TAP spec indicates why, though:

> A TAP parser is required to not consider an unknown line as an error but
> may optionally choose to capture said line and hand it to the test
> harness, which may have custom behavior attached. This is to allow for
> forward compatability. Test::Harness silently ignores incorrect lines,
> but will become more stringent in the future. TAP::Harness reports TAP
> syntax errors at the end of a test run.

The goal of treating unknown lines as an error in the TAP parser is not
because unknown lines are fine and dandy. The goal is to allow
implementing future versions of TAP, and handling it via existing
parsers. Since Meson has both a parser and a harness, let's do exactly
that -- pass these lines as a distinctive status to the test harness,
then have the test harness complain.

mesonbuild/mtest.py

@@ -275,6 +275,7 @@ TYPE_TAPResult = T.Union['TAPParser.Test',
                          'TAPParser.Error',
                          'TAPParser.Version',
                          'TAPParser.Plan',
+                         'TAPParser.UnknownLine',
                          'TAPParser.Bailout']
 
 class TAPParser:
@@ -299,6 +300,10 @@ class TAPParser:
     class Error(T.NamedTuple):
         message: str
 
+    class UnknownLine(T.NamedTuple):
+        message: str
+        lineno: int
+
     class Version(T.NamedTuple):
         version: int
 
@@ -434,6 +439,9 @@ class TAPParser:
                 else:
                     yield self.Version(version=self.version)
                 return
+
+            # unknown syntax
+            yield self.UnknownLine(line, self.lineno)
         else:
             # end of file
             if self.state == self._YAML:
@@ -673,6 +681,11 @@ class ConsoleLogger(TestLogger):
                       flush=True)
                 if result.verbose or result.res.is_bad():
                     self.print_log(harness, result)
+            if result.warnings:
+                print(flush=True)
+                for w in result.warnings:
+                    print(w, flush=True)
+                print(flush=True)
             if result.verbose or result.res.is_bad():
                 print(flush=True)
 
@@ -899,6 +912,7 @@ class TestRun:
         self.junit = None      # type: T.Optional[et.ElementTree]
         self.is_parallel = is_parallel
         self.verbose = verbose
+        self.warnings = []     # type: T.List[str]
 
     def start(self, cmd: T.List[str]) -> None:
         self.res = TestResult.RUNNING
@@ -1041,9 +1055,13 @@ class TestRunTAP(TestRun):
 
     async def parse(self, harness: 'TestHarness', lines: T.AsyncIterator[str]) -> None:
         res = None
+        warnings = [] # type: T.List[TAPParser.UnknownLine]
+        version: int
 
         async for i in TAPParser().parse_async(lines):
-            if isinstance(i, TAPParser.Bailout):
+            if isinstance(i, TAPParser.Version):
+                version = i.version
+            elif isinstance(i, TAPParser.Bailout):
                 res = TestResult.ERROR
                 harness.log_subtest(self, i.message, res)
             elif isinstance(i, TAPParser.Test):
@@ -1051,10 +1069,23 @@ class TestRunTAP(TestRun):
                 if i.result.is_bad():
                     res = TestResult.FAIL
                 harness.log_subtest(self, i.name or f'subtest {i.number}', i.result)
+            elif isinstance(i, TAPParser.UnknownLine):
+                warnings.append(i)
             elif isinstance(i, TAPParser.Error):
                 self.additional_error += 'TAP parsing error: ' + i.message
                 res = TestResult.ERROR
 
+        if warnings:
+            unknown = str(mlog.yellow('UNKNOWN'))
+            width = len(str(max(i.lineno for i in warnings)))
+            for w in warnings:
+                self.warnings.append(f'stdout: {w.lineno:{width}}: {unknown}: {w.message}')
+            if version > 13:
+                self.warnings.append('Unknown TAP output lines have been ignored. Please open a feature request to\n'
+                                     'implement them, or prefix them with a # if they are not TAP syntax.')
+            else:
+                self.warnings.append(str(mlog.red('ERROR')) + ': Unknown TAP output lines for a supported TAP version.\n'
+                                     'This is probably a bug in the test; if they are not TAP syntax, prefix them with a #')
         if all(t.result is TestResult.SKIP for t in self.results):
             # This includes the case where self.results is empty
             res = TestResult.SKIP

unittests/taptests.py

@@ -37,6 +37,9 @@ class TAPParserTests(unittest.TestCase):
     def assert_error(self, events):
         self.assertEqual(type(next(events)), TAPParser.Error)
 
+    def assert_unexpected(self, events, **kwargs):
+        self.assertEqual(next(events), TAPParser.UnknownLine(**kwargs))
+
     def assert_bailout(self, events, **kwargs):
         self.assertEqual(next(events), TAPParser.Bailout(**kwargs))
 
@@ -255,6 +258,7 @@ class TAPParserTests(unittest.TestCase):
     def test_unexpected(self):
         events = self.parse_tap('1..1\ninvalid\nok 1')
         self.assert_plan(events, num_tests=1, late=False)
+        self.assert_unexpected(events, message='invalid', lineno=2)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)