mtest: create separate runners for multiple repeats

Reusing the runners for multiple repeats of the test run gets in the
way of the progress report, which stores runners in an OrderedSet.
Instead, create a separate SingleTestRunner object for each repeat.

While at it, fix the "duplicate suite" assertion as it can fire
with TAP tests and --repeat=N.

Fixes: #8405

mesonbuild/mtest.py

@@ -774,7 +774,7 @@ class JunitBuilder(TestLogger):
         # separately
         if test.results:
             suitename = '{}.{}'.format(test.project, test.name)
-            assert suitename not in self.suites, 'duplicate suite'
+            assert suitename not in self.suites or harness.options.repeat > 1, 'duplicate suite'
 
             suite = self.suites[suitename] = et.Element(
                 'testsuite',
@@ -1627,18 +1627,22 @@ class TestHarness:
             # wrapper script.
             sys.exit(125)
 
-        self.test_count = len(tests)
         self.name_max_len = max([uniwidth(self.get_pretty_suite(test)) for test in tests])
         startdir = os.getcwd()
         try:
             if self.options.wd:
                 os.chdir(self.options.wd)
-            runners = [self.get_test_runner(test) for test in tests]
-            self.duration_max_len = max([len(str(int(runner.timeout or 99)))
-                                         for runner in runners])
-            # Disable the progress report if it gets in the way
-            self.need_console = any((runner.console_mode is not ConsoleUser.LOGGER
-                                     for runner in runners))
+            runners = []             # type: T.List[SingleTestRunner]
+            for i in range(self.options.repeat):
+                runners.extend((self.get_test_runner(test) for test in tests))
+                if i == 0:
+                    self.duration_max_len = max([len(str(int(runner.timeout or 99)))
+                                                 for runner in runners])
+                    # Disable the progress report if it gets in the way
+                    self.need_console = any((runner.console_mode is not ConsoleUser.LOGGER
+                                             for runner in runners))
+
+            self.test_count = len(runners)
             self.run_tests(runners)
         finally:
             os.chdir(startdir)
@@ -1860,16 +1864,15 @@ class TestHarness:
             asyncio.get_event_loop().add_signal_handler(signal.SIGINT, sigint_handler)
             asyncio.get_event_loop().add_signal_handler(signal.SIGTERM, sigterm_handler)
         try:
-            for _ in range(self.options.repeat):
-                for runner in runners:
-                    if not runner.is_parallel:
-                        await complete_all(futures)
-                    future = asyncio.ensure_future(run_test(runner))
-                    futures.append(future)
-                    running_tests[future] = runner.visible_name
-                    future.add_done_callback(test_done)
-                    if not runner.is_parallel:
-                        await complete(future)
+            for runner in runners:
+                if not runner.is_parallel:
+                    await complete_all(futures)
+                future = asyncio.ensure_future(run_test(runner))
+                futures.append(future)
+                running_tests[future] = runner.visible_name
+                future.add_done_callback(test_done)
+                if not runner.is_parallel:
+                    await complete(future)
                 if self.options.repeat > 1 and self.fail_count:
                     break
 

run_unittests.py

@@ -2524,6 +2524,12 @@ class AllPlatformTests(BasePlatformTests):
         self.assertNotRegex(out, r'WARNING: Overriding.*TEST_VAR_SET')
         self.run_tests()
 
+    def test_testrepeat(self):
+        testdir = os.path.join(self.common_test_dir, '207 tap tests')
+        self.init(testdir)
+        self.build()
+        self._run(self.mtest_command + ['--repeat=2'])
+
     def test_testsetups(self):
         if not shutil.which('valgrind'):
             raise unittest.SkipTest('Valgrind not installed.')