Merge pull request #496 from mesonbuild/partest

Run tests in parallel

mesonbuild/environment.py

@@ -58,6 +58,7 @@ class Environment():
     log_dir = 'meson-logs'
     coredata_file = os.path.join(private_dir, 'coredata.dat')
     version_regex = '\d+(\.\d+)+(-[a-zA-Z0-9]+)?'
+
     def __init__(self, source_dir, build_dir, main_script_file, options, original_cmd_line_args):
         assert(os.path.isabs(main_script_file))
         assert(not os.path.islink(main_script_file))

run_tests.py

@@ -28,6 +28,8 @@ from mesonbuild.scripts import meson_test, meson_benchmark
 import argparse
 import xml.etree.ElementTree as ET
 import time
+import multiprocessing
+import concurrent.futures as conc
 
 from mesonbuild.mesonmain import backendlist
 
@@ -195,7 +197,9 @@ def parse_test_args(testdir):
         pass
     return args
 
-def run_test(testdir, extra_args, should_succeed):
+def run_test(skipped, testdir, extra_args, should_succeed):
+    if skipped:
+        return None
     with tempfile.TemporaryDirectory(prefix='b ', dir='.') as build_dir:
         with tempfile.TemporaryDirectory(prefix='i ', dir=os.getcwd()) as install_dir:
             try:
@@ -205,7 +209,6 @@ def run_test(testdir, extra_args, should_succeed):
 
 def _run_test(testdir, test_build_dir, install_dir, extra_args, should_succeed):
     global compile_commands
-    print('Running test: ' + testdir)
     test_args = parse_test_args(testdir)
     gen_start = time.time()
     gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\
@@ -292,34 +295,51 @@ def run_tests(extra_args):
     build_time = 0
     test_time = 0
 
+    executor = conc.ProcessPoolExecutor(max_workers=multiprocessing.cpu_count())
+
     for name, test_cases, skipped in all_tests:
         current_suite = ET.SubElement(junit_root, 'testsuite', {'name' : name, 'tests' : str(len(test_cases))})
         if skipped:
             print('\nNot running %s tests.\n' % name)
         else:
             print('\nRunning %s tests.\n' % name)
+        futures = []
         for t in test_cases:
             # Jenkins screws us over by automatically sorting test cases by name
             # and getting it wrong by not doing logical number sorting.
             (testnum, testbase) = os.path.split(t)[-1].split(' ', 1)
             testname = '%.3d %s' % (int(testnum), testbase)
-            if skipped:
+            # Windows errors out when calling result.result() below with
+            # a bizarre error about appending None to an array that comes
+            # from the standard library. This is probably either because I use
+            # XP or the Python version is old. Anyhow, fall back to immediate
+            # evaluation. This causes output not to be printed until the end,
+            # which is unfortunate but least it works.
+            if mesonlib.is_windows():
+                result = run_test(skipped, t, extra_args, name != 'failing')
+            else:
+                result = executor.submit(run_test, skipped, t, extra_args, name != 'failing')
+            futures.append((testname, t, result))
+        for (testname, t, result) in futures:
+            if not mesonlib.is_windows(): # See above.
+                result = result.result()
+            if result is None:
+                print('Skipping:', t)
                 current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname,
                                                                          'classname' : name})
                 ET.SubElement(current_test, 'skipped', {})
                 global skipped_tests
                 skipped_tests += 1
             else:
-                ts = time.time()
+                print('Running test: ' + t)
-                result = run_test(t, extra_args, name != 'failing')
-                te = time.time()
                 conf_time += result.conftime
                 build_time += result.buildtime
                 test_time += result.testtime
+                total_time = conf_time + build_time + test_time
                 log_text_file(logfile, t, result.msg, result.stdo, result.stde)
                 current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname,
                                                                          'classname' : name,
-                                                                         'time' : '%.3f' % (te - ts)})
+                                                                         'time' : '%.3f' % total_time})
                 if result.msg != '':
                     ET.SubElement(current_test, 'failure', {'message' : result.msg})
                 stdoel = ET.SubElement(current_test, 'system-out')