meson/mesontest.py

#!/usr/bin/env python3

# Copyright 2016 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# A tool to run tests in many different ways.

import subprocess, sys, os, argparse
import pickle
import mesonbuild
from mesonbuild import build
from mesonbuild import environment

import time, datetime, pickle, multiprocessing, json
import concurrent.futures as conc
import platform
import signal

def is_windows():
    platname = platform.system().lower()
    return platname == 'windows' or 'mingw' in platname

def determine_worker_count():
    varname = 'MESON_TESTTHREADS'
    if varname in os.environ:
        try:
            num_workers = int(os.environ[varname])
        except ValueError:
            print('Invalid value in %s, using 1 thread.' % varname)
            num_workers = 1
    else:
        try:
            # Fails in some weird environments such as Debian
            # reproducible build.
            num_workers = multiprocessing.cpu_count()
        except Exception:
            num_workers = 1
    return num_workers

parser = argparse.ArgumentParser()
parser.add_argument('--repeat', default=1, dest='repeat', type=int,
                    help='Number of times to run the tests.')
parser.add_argument('--gdb', default=False, dest='gdb', action='store_true',
                    help='Run test under gdb.')
parser.add_argument('--list', default=False, dest='list', action='store_true',
                    help='List available tests.')
parser.add_argument('--wrapper', default=None, dest='wrapper',
                    help='wrapper to run tests with (e.g. Valgrind)')
parser.add_argument('--wd', default=None, dest='wd',
                    help='directory to cd into before running')
parser.add_argument('--suite', default=None, dest='suite',
                    help='Only run tests belonging to the given suite.')
parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
                    help='Do not split stderr and stdout in test logs.')
parser.add_argument('--print-errorlogs', default=False, action='store_true',
                    help="Whether to print faling tests' logs.")
parser.add_argument('--benchmark', default=False, action='store_true',
                    help="Run benchmarks instead of tests.")
parser.add_argument('--logbase', default='testlog',
                    help="Base name for log file.")
parser.add_argument('--num-processes', default=determine_worker_count(), type=int,
                    help='How many parallel processes to use.')
parser.add_argument('args', nargs='*')

class TestRun():
    def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd,
                 env):
        self.res = res
        self.returncode = returncode
        self.duration = duration
        self.stdo = stdo
        self.stde = stde
        self.cmd = cmd
        self.env = env
        self.should_fail = should_fail

    def get_log(self):
        res = '--- command ---\n'
        if self.cmd is None:
            res += 'NONE\n'
        else:
            res += "\n%s %s\n" %(' '.join(
                ["%s='%s'" % (k, v) for k, v in self.env.items()]),
                ' ' .join(self.cmd))
        if self.stdo:
            res += '--- stdout ---\n'
            res += self.stdo
        if self.stde:
            if res[-1:] != '\n':
                res += '\n'
            res += '--- stderr ---\n'
            res += self.stde
        if res[-1:] != '\n':
            res += '\n'
        res += '-------\n\n'
        return res

def decode(stream):
    try:
        return stream.decode('utf-8')
    except UnicodeDecodeError:
        return stream.decode('iso-8859-1', errors='ignore')

def write_json_log(jsonlogfile, test_name, result):
    jresult = {'name' : test_name,
              'stdout' : result.stdo,
              'result' : result.res,
              'duration' : result.duration,
              'returncode' : result.returncode,
              'command' : result.cmd,
              'env' : result.env}
    if result.stde:
        jresult['stderr'] = result.stde
    jsonlogfile.write(json.dumps(jresult) + '\n')

def run_with_mono(fname):
    if fname.endswith('.exe') and not is_windows():
        return True
    return False

class TestHarness:
    def __init__(self, options):
        self.options = options
        self.collected_logs = []
        self.error_count = 0
        self.is_run = False
        if self.options.benchmark:
            self.datafile = 'meson-private/meson_benchmark_setup.dat'
        else:
            self.datafile = 'meson-private/meson_test_setup.dat'

    def run_single_test(self, wrap, test):
        if test.fname[0].endswith('.jar'):
            cmd = ['java', '-jar'] + test.fname
        elif not test.is_cross and run_with_mono(test.fname[0]):
            cmd = ['mono'] + test.fname
        else:
            if test.is_cross:
                if test.exe_runner is None:
                    # Can not run test on cross compiled executable
                    # because there is no execute wrapper.
                    cmd = None
                else:
                    cmd = [test.exe_runner] + test.fname
            else:
                cmd = test.fname
        if cmd is None:
            res = 'SKIP'
            duration = 0.0
            stdo = 'Not run because can not execute cross compiled binaries.'
            stde = None
            returncode = -1
        else:
            cmd = wrap + cmd + test.cmd_args
            starttime = time.time()
            child_env = os.environ.copy()
            if isinstance(test.env, build.EnvironmentVariables):
                test.env = test.env.get_env(child_env)

            child_env.update(test.env)
            if len(test.extra_paths) > 0:
                child_env['PATH'] = child_env['PATH'] + ';'.join([''] + test.extra_paths)
            if is_windows():
                setsid = None
            else:
                setsid = os.setsid
            p = subprocess.Popen(cmd,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT,
                                 env=child_env,
                                 cwd=test.workdir,
                                 preexec_fn=setsid)
            timed_out = False
            try:
                (stdo, stde) = p.communicate(timeout=test.timeout)
            except subprocess.TimeoutExpired:
                timed_out = True
                # Python does not provide multiplatform support for
                # killing a process and all its children so we need
                # to roll our own.
                if is_windows():
                    subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)])
                else:
                    os.killpg(os.getpgid(p.pid), signal.SIGKILL)
                (stdo, stde) = p.communicate()
            endtime = time.time()
            duration = endtime - starttime
            stdo = decode(stdo)
            if stde:
                stde = decode(stde)
            if timed_out:
                res = 'TIMEOUT'
            elif (not test.should_fail and p.returncode == 0) or \
                (test.should_fail and p.returncode != 0):
                res = 'OK'
            else:
                res = 'FAIL'
            returncode = p.returncode
        return TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env)

    def print_stats(self, numlen, tests, name, result, i, logfile, jsonlogfile):
        startpad = ' '*(numlen - len('%d' % (i+1)))
        num = '%s%d/%d' % (startpad, i+1, len(tests))
        padding1 = ' '*(38-len(name))
        padding2 = ' '*(8-len(result.res))
        result_str = '%s %s  %s%s%s%5.2f s' % \
            (num, name, padding1, result.res, padding2, result.duration)
        print(result_str)
        result_str += "\n\n" + result.get_log()
        if (result.returncode != 0) != result.should_fail:
            self.error_count += 1
            if self.options.print_errorlogs:
                self.collected_logs.append(result_str)
        logfile.write(result_str)
        write_json_log(jsonlogfile, name, result)

    def doit(self):
        if self.is_run:
            raise RuntimeError('Test harness object can only be used once.')
        if not os.path.isfile(self.datafile):
            print('Test data file. Probably this means that you did not run this in the build directory.')
            return 1
        self.is_run = True
        logfilename = self.run_tests(self.datafile, self.options.logbase)
        if len(self.collected_logs) > 0:
            if len(self.collected_logs) > 10:
                print('\nThe output from 10 first failed tests:\n')
            else:
                print('\nThe output from the failed tests:\n')
            for log in self.collected_logs[:10]:
                lines = log.splitlines()
                if len(lines) > 100:
                    print(lines[0])
                    print('--- Listing only the last 100 lines from a long log. ---')
                    lines = lines[-99:]
                for line in lines:
                    print(line)
        print('Full log written to %s.' % logfilename)
        return self.error_count

    def run_tests(self, datafilename, log_base):
        logfile_base = os.path.join('meson-logs', log_base)
        if self.options.wrapper is None:
            wrap = []
            logfilename = logfile_base + '.txt'
            jsonlogfilename = logfile_base+ '.json'
        else:
            wrap = self.options.wrapper.split()
            namebase = wrap[0]
            logfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.txt'
            jsonlogfilename = logfile_base + '-' + namebase.replace(' ', '_') + '.json'
        with open(datafilename, 'rb') as f:
            tests = pickle.load(f)
        if len(tests) == 0:
            print('No tests defined.')
            return
        numlen = len('%d' % len(tests))
        executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes)
        futures = []
        filtered_tests = filter_tests(self.options.suite, tests)

        with open(jsonlogfilename, 'w') as jsonlogfile, \
             open(logfilename, 'w') as logfile:
            logfile.write('Log of Meson test suite run on %s.\n\n' %
                          datetime.datetime.now().isoformat())
            for i, test in enumerate(filtered_tests):
                if test.suite[0] == '':
                    visible_name = test.name
                else:
                    if self.options.suite is not None:
                        visible_name = self.options.suite + ' / ' + test.name
                    else:
                        visible_name = test.suite[0] + ' / ' + test.name

                if not test.is_parallel:
                    self.drain_futures(futures)
                    futures = []
                    res = self.run_single_test(wrap, test)
                    self.print_stats(numlen, filtered_tests, visible_name, res, i,
                                     logfile, jsonlogfile)
                else:
                    f = executor.submit(self.run_single_test, wrap, test)
                    futures.append((f, numlen, filtered_tests, visible_name, i,
                                    logfile, jsonlogfile))
            self.drain_futures(futures)
        return logfilename


    def drain_futures(self, futures):
        for i in futures:
            (result, numlen, tests, name, i, logfile, jsonlogfile) = i
            self.print_stats(numlen, tests, name, result.result(), i, logfile, jsonlogfile)

    def run_special(self):
        'Tests run by the user, usually something like "under gdb 1000 times".'
        if self.is_run:
            raise RuntimeError('Can not use run_special after a full run.')
        if self.options.wrapper is not None:
            wrap = self.options.wrapper.split(' ')
        else:
            wrap = []
        if self.options.gdb and len(wrap) > 0:
            print('Can not specify both a wrapper and gdb.')
            return 1
        if os.path.isfile('build.ninja'):
            subprocess.check_call([environment.detect_ninja(), 'all'])
        tests = pickle.load(open(self.datafile, 'rb'))
        if self.options.list:
            for i in tests:
                print(i.name)
            return 0
        for t in tests:
            if t.name in self.options.args:
                for i in range(self.options.repeat):
                    print('Running: %s %d/%d' % (t.name, i+1, self.options.repeat))
                    if self.options.gdb:
                        gdbrun(t)
                    else:
                        res = self.run_single_test(wrap, t)
                        if (res.returncode == 0 and res.should_fail) or \
                            (res.returncode != 0 and not res.should_fail):
                            print('Test failed:\n\n-- stdout --\n')
                            print(res.stdo)
                            print('\n-- stderr --\n')
                            print(res.stde)
                            return 1
        return 0

def filter_tests(suite, tests):
    if suite is None:
        return tests
    return [x for x in tests if suite in x.suite]

def gdbrun(test):
    child_env = os.environ.copy()
    child_env.update(test.env)
    # On success will exit cleanly. On failure gdb will ask user
    # if they really want to exit.
    exe = test.fname
    args = test.cmd_args
    if len(args) > 0:
        argset = ['-ex', 'set args ' + ' '.join(args)]
    else:
        argset = []
    cmd = ['gdb', '--quiet'] + argset + ['-ex', 'run', '-ex', 'quit'] + exe
    # FIXME a ton of stuff. run_single_test grabs stdout & co,
    # which we do not want to do when running under gdb.
    p = subprocess.Popen(cmd,
                         env=child_env,
                         cwd=test.workdir,
                         )
    p.communicate()

def run(args):
    options = parser.parse_args(args)
    if options.benchmark:
        options.num_processes = 1
    th = TestHarness(options)
    if len(options.args) == 0:
        return th.doit()
    return th.run_special()

if __name__ == '__main__':
    sys.exit(run(sys.argv[1:]))