#!/usr/bin/env python3 # Copyright 2012-2016 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from glob import glob import os, subprocess, shutil, sys, signal from io import StringIO from ast import literal_eval import sys, tempfile from mesonbuild import environment from mesonbuild import mesonlib from mesonbuild import mlog from mesonbuild import mesonmain from mesonbuild.mesonlib import stringlistify from mesonbuild.scripts import meson_test, meson_benchmark import argparse import xml.etree.ElementTree as ET import time import multiprocessing import concurrent.futures as conc from mesonbuild.mesonmain import backendlist class TestResult: def __init__(self, msg, stdo, stde, conftime=0, buildtime=0, testtime=0): self.msg = msg self.stdo = stdo self.stde = stde self.conftime = conftime self.buildtime = buildtime self.testtime = testtime passing_tests = 0 failing_tests = 0 skipped_tests = 0 print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ meson_command = os.path.join(os.getcwd(), 'meson') if not os.path.exists(meson_command): meson_command += '.py' if not os.path.exists(meson_command): raise RuntimeError('Could not find main Meson script to run.') class StopException(Exception): def __init__(self): super().__init__('Stopped by user') stop = False def stop_handler(signal, frame): global stop stop = True signal.signal(signal.SIGINT, stop_handler) signal.signal(signal.SIGTERM, stop_handler) #unity_flags = ['--unity'] unity_flags = [] backend_flags = None compile_commands = None test_commands = None install_commands = None def setup_commands(backend): global backend_flags, compile_commands, test_commands, install_commands msbuild_exe = shutil.which('msbuild') if backend == 'vs2010' or (backend is None and msbuild_exe is not None): backend_flags = ['--backend=vs2010'] compile_commands = ['msbuild'] test_commands = ['msbuild', 'RUN_TESTS.vcxproj'] install_commands = [] elif backend == 'xcode' or (backend is None and mesonlib.is_osx()): backend_flags = ['--backend=xcode'] compile_commands = ['xcodebuild'] test_commands = ['xcodebuild', '-target', 'RUN_TESTS'] install_commands = [] else: backend_flags = [] ninja_command = environment.detect_ninja() if ninja_command is None: raise RuntimeError('Could not find Ninja executable.') if print_debug: compile_commands = [ninja_command, '-v'] else: compile_commands = [ninja_command] test_commands = [ninja_command, 'test', 'benchmark'] install_commands = [ninja_command, 'install'] def platform_fix_filename(fname): if mesonlib.is_osx(): if fname.endswith('.so'): return fname[:-2] + 'dylib' return fname.replace('.so.', '.dylib.') elif mesonlib.is_windows(): if fname.endswith('.so'): (p, f) = os.path.split(fname) f = f[3:-2] + 'dll' return os.path.join(p, f) if fname.endswith('.a'): return fname[:-1] + 'lib' return fname def validate_install(srcdir, installdir): if mesonlib.is_windows(): # Don't really know how Windows installs should work # so skip. return '' info_file = os.path.join(srcdir, 'installed_files.txt') expected = {} found = {} if os.path.exists(info_file): for line in open(info_file): expected[platform_fix_filename(line.strip())] = True for root, _, files in os.walk(installdir): for fname in files: found_name = os.path.join(root, fname)[len(installdir)+1:] found[found_name] = True expected = set(expected) found = set(found) missing = expected - found for fname in missing: return 'Expected file %s missing.' % fname extra = found - expected for fname in extra: return 'Found extra file %s.' % fname return '' def log_text_file(logfile, testdir, msg, stdo, stde): global passing_tests, failing_tests, stop if msg != '': print('Fail:', msg) failing_tests += 1 else: print('Success') passing_tests += 1 logfile.write('%s\nstdout\n\n---\n' % testdir) logfile.write(stdo) logfile.write('\n\n---\n\nstderr\n\n---\n') logfile.write(stde) logfile.write('\n\n---\n\n') if print_debug: print(stdo) print(stde, file=sys.stderr) if stop: raise StopException() def run_configure_inprocess(commandlist): old_stdout = sys.stdout sys.stdout = mystdout = StringIO() old_stderr = sys.stderr sys.stderr = mystderr = StringIO() try: returncode = mesonmain.run(commandlist[0], commandlist[1:]) finally: sys.stdout = old_stdout sys.stderr = old_stderr return (returncode, mystdout.getvalue(), mystderr.getvalue()) def run_test_inprocess(testdir): old_stdout = sys.stdout sys.stdout = mystdout = StringIO() old_stderr = sys.stderr sys.stderr = mystderr = StringIO() old_cwd = os.getcwd() os.chdir(testdir) try: returncode_test = meson_test.run(['meson-private/meson_test_setup.dat']) returncode_benchmark = meson_benchmark.run(['meson-private/meson_benchmark_setup.dat']) finally: sys.stdout = old_stdout sys.stderr = old_stderr os.chdir(old_cwd) return (max(returncode_test, returncode_benchmark), mystdout.getvalue(), mystderr.getvalue()) def parse_test_args(testdir): args = [] try: with open(os.path.join(testdir, 'test_args.txt'), 'r') as f: content = f.read() try: args = literal_eval(content) except Exception: raise Exception('Malformed test_args file.') args = stringlistify(args) except FileNotFoundError: pass return args def run_test(skipped, testdir, extra_args, should_succeed): if skipped: return None with tempfile.TemporaryDirectory(prefix='b ', dir='.') as build_dir: with tempfile.TemporaryDirectory(prefix='i ', dir=os.getcwd()) as install_dir: try: return _run_test(testdir, build_dir, install_dir, extra_args, should_succeed) finally: mlog.shutdown() # Close the log file because otherwise Windows wets itself. def _run_test(testdir, test_build_dir, install_dir, extra_args, should_succeed): global compile_commands test_args = parse_test_args(testdir) gen_start = time.time() gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\ + unity_flags + backend_flags + test_args + extra_args (returncode, stdo, stde) = run_configure_inprocess(gen_command) gen_time = time.time() - gen_start if not should_succeed: if returncode != 0: return TestResult('', stdo, stde, gen_time) return TestResult('Test that should have failed succeeded', stdo, stde, gen_time) if returncode != 0: return TestResult('Generating the build system failed.', stdo, stde, gen_time) if 'msbuild' in compile_commands[0]: sln_name = glob(os.path.join(test_build_dir, '*.sln'))[0] comp = compile_commands + [os.path.split(sln_name)[-1]] else: comp = compile_commands build_start = time.time() pc = subprocess.Popen(comp, cwd=test_build_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (o, e) = pc.communicate() build_time = time.time() - build_start stdo += o.decode(sys.stdout.encoding) stde += e.decode(sys.stdout.encoding) if pc.returncode != 0: return TestResult('Compiling source code failed.', stdo, stde, gen_time, build_time) test_start = time.time() # Note that we don't test that running e.g. 'ninja test' actually # works. One hopes that this is a common enough happening that # it is picked up immediately on development. (returncode, tstdo, tstde) = run_test_inprocess(test_build_dir) test_time = time.time() - test_start stdo += tstdo stde += tstde if returncode != 0: return TestResult('Running unit tests failed.', stdo, stde, gen_time, build_time, test_time) if len(install_commands) == 0: print("Skipping install test") return TestResult('', '', '', gen_time, build_time, test_time) else: env = os.environ.copy() env['DESTDIR'] = install_dir pi = subprocess.Popen(install_commands, cwd=test_build_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (o, e) = pi.communicate() stdo += o.decode(sys.stdout.encoding) stde += e.decode(sys.stdout.encoding) if pi.returncode != 0: return TestResult('Running install failed.', stdo, stde, gen_time, build_time, test_time) return TestResult(validate_install(testdir, install_dir), stdo, stde, gen_time, build_time, test_time) def gather_tests(testdir): tests = [t.replace('\\', '/').split('/', 2)[2] for t in glob(os.path.join(testdir, '*'))] testlist = [(int(t.split()[0]), t) for t in tests] testlist.sort() tests = [os.path.join(testdir, t[1]) for t in testlist] return tests def detect_tests_to_run(): all_tests = [] all_tests.append(('common', gather_tests('test cases/common'), False)) all_tests.append(('failing', gather_tests('test cases/failing'), False)) all_tests.append(('prebuilt object', gather_tests('test cases/prebuilt object'), False)) all_tests.append(('platform-osx', gather_tests('test cases/osx'), False if mesonlib.is_osx() else True)) all_tests.append(('platform-windows', gather_tests('test cases/windows'), False if mesonlib.is_windows() else True)) all_tests.append(('platform-linux', gather_tests('test cases/linuxlike'), False if not (mesonlib.is_osx() or mesonlib.is_windows()) else True)) all_tests.append(('framework', gather_tests('test cases/frameworks'), False if not mesonlib.is_osx() and not mesonlib.is_windows() else True)) all_tests.append(('java', gather_tests('test cases/java'), False if not mesonlib.is_osx() and shutil.which('javac') else True)) all_tests.append(('C#', gather_tests('test cases/csharp'), False if shutil.which('mcs') else True)) all_tests.append(('vala', gather_tests('test cases/vala'), False if shutil.which('valac') else True)) all_tests.append(('rust', gather_tests('test cases/rust'), False if shutil.which('rustc') else True)) all_tests.append(('objective c', gather_tests('test cases/objc'), False if not mesonlib.is_windows() else True)) all_tests.append(('fortran', gather_tests('test cases/fortran'), False if shutil.which('gfortran') else True)) all_tests.append(('swift', gather_tests('test cases/swift'), False if shutil.which('swiftc') else True)) all_tests.append(('python3', gather_tests('test cases/python3'), False if shutil.which('python3') else True)) return all_tests def run_tests(extra_args): all_tests = detect_tests_to_run() logfile = open('meson-test-run.txt', 'w', encoding="utf_8") junit_root = ET.Element('testsuites') conf_time = 0 build_time = 0 test_time = 0 executor = conc.ProcessPoolExecutor(max_workers=multiprocessing.cpu_count()) for name, test_cases, skipped in all_tests: current_suite = ET.SubElement(junit_root, 'testsuite', {'name' : name, 'tests' : str(len(test_cases))}) if skipped: print('\nNot running %s tests.\n' % name) else: print('\nRunning %s tests.\n' % name) futures = [] for t in test_cases: # Jenkins screws us over by automatically sorting test cases by name # and getting it wrong by not doing logical number sorting. (testnum, testbase) = os.path.split(t)[-1].split(' ', 1) testname = '%.3d %s' % (int(testnum), testbase) # Windows errors out when calling result.result() below with # a bizarre error about appending None to an array that comes # from the standard library. This is probably either because I use # XP or the Python version is old. Anyhow, fall back to immediate # evaluation. This causes output not to be printed until the end, # which is unfortunate but least it works. if mesonlib.is_windows(): result = run_test(skipped, t, extra_args, name != 'failing') else: result = executor.submit(run_test, skipped, t, extra_args, name != 'failing') futures.append((testname, t, result)) for (testname, t, result) in futures: if not mesonlib.is_windows(): # See above. result = result.result() if result is None: print('Skipping:', t) current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname, 'classname' : name}) ET.SubElement(current_test, 'skipped', {}) global skipped_tests skipped_tests += 1 else: print('Running test: ' + t) conf_time += result.conftime build_time += result.buildtime test_time += result.testtime total_time = conf_time + build_time + test_time log_text_file(logfile, t, result.msg, result.stdo, result.stde) current_test = ET.SubElement(current_suite, 'testcase', {'name' : testname, 'classname' : name, 'time' : '%.3f' % total_time}) if result.msg != '': ET.SubElement(current_test, 'failure', {'message' : result.msg}) stdoel = ET.SubElement(current_test, 'system-out') stdoel.text = result.stdo stdeel = ET.SubElement(current_test, 'system-err') stdeel.text = result.stde print("\nTotal configuration time: %.2fs" % conf_time) print("Total build time: %.2fs" % build_time) print("Total test time: %.2fs" % test_time) ET.ElementTree(element=junit_root).write('meson-test-run.xml', xml_declaration=True, encoding='UTF-8') def check_file(fname): linenum = 1 for line in open(fname, 'rb').readlines(): if b'\t' in line: print("File %s contains a literal tab on line %d. Only spaces are permitted." % (fname, linenum)) sys.exit(1) if b'\r' in line: print("File %s contains DOS line ending on line %d. Only unix-style line endings are permitted." % (fname, linenum)) sys.exit(1) linenum += 1 def check_format(): for (root, _, files) in os.walk('.'): for file in files: if file.endswith('.py') or file.endswith('.build') or file == 'meson_options.txt': fullname = os.path.join(root, file) check_file(fullname) def generate_prebuilt_object(): source = 'test cases/prebuilt object/1 basic/source.c' objectbase = 'test cases/prebuilt object/1 basic/prebuilt.' if shutil.which('cl'): objectfile = objectbase + 'obj' cmd = ['cl', '/nologo', '/Fo'+objectfile, '/c', source] else: if mesonlib.is_windows(): objectfile = objectbase + 'obj' else: objectfile = objectbase + 'o' if shutil.which('cc'): cmd = 'cc' elif shutil.which('gcc'): cmd = 'gcc' else: raise RuntimeError("Could not find C compiler.") cmd = [cmd, '-c', source, '-o', objectfile] subprocess.check_call(cmd) return objectfile if __name__ == '__main__': parser = argparse.ArgumentParser(description="Run the test suite of Meson.") parser.add_argument('extra_args', nargs='*', help='arguments that are passed directly to Meson (remember to have -- before these).') parser.add_argument('--backend', default=None, dest='backend', choices = backendlist) options = parser.parse_args() setup_commands(options.backend) script_dir = os.path.split(__file__)[0] if script_dir != '': os.chdir(script_dir) check_format() pbfile = generate_prebuilt_object() try: run_tests(options.extra_args) except StopException: pass os.unlink(pbfile) print('\nTotal passed tests:', passing_tests) print('Total failed tests:', failing_tests) print('Total skipped tests:', skipped_tests) sys.exit(failing_tests)