diff --git a/docs/markdown/Release-notes-for-0.42.0.md b/docs/markdown/Release-notes-for-0.42.0.md index 7eb603d0e..54723410c 100644 --- a/docs/markdown/Release-notes-for-0.42.0.md +++ b/docs/markdown/Release-notes-for-0.42.0.md @@ -83,12 +83,12 @@ flag manually, e.g. via `link_args` to a target. This is not recommended because having multiple rpath causes them to stomp on each other. This warning will become a hard error in some future release. - ## Vulkan dependency module -Vulkan can now be used as native dependency. The dependency module will detect -the VULKAN_SDK environment variable or otherwise try to receive the vulkan -library and header via pkgconfig or from the system. +Vulkan can now be used as native dependency. The dependency module +will detect the VULKAN_SDK environment variable or otherwise try to +receive the vulkan library and header via pkgconfig or from the +system. ## Limiting the maximum number of linker processes @@ -120,3 +120,14 @@ accepts the new `exclude_files` and `exclude_directories` keyword arguments that allow specified files or directories to be excluded from the installed subdirectory. +## Make all Meson functionality invokable via the main executable + +Previously Meson had multiple executables such as `mesonintrospect` +and `mesontest`. They are now invokable via the main Meson executable +like this: + + meson configure # equivalent to mesonconf + meson test # equivalent to mesontest + +The old commands are still available but they are deprecated +and will be removed in some future release. diff --git a/man/meson.1 b/man/meson.1 index 1cd60b678..086f88bf8 100644 --- a/man/meson.1 +++ b/man/meson.1 @@ -8,11 +8,16 @@ productivity. It aims to do this by providing simple, out-of-the-box support for modern software development tools and practices, such as unit tests, coverage reports, Valgrind, CCache and the like. +The main Meson executable provides many subcommands to access all +the functionality. + +.SH The setup command + Using Meson is simple and follows the common two-phase process of most build systems. First you run Meson to configure your build: -.B meson [ +.B meson setup [ .I options .B ] [ .I source directory @@ -35,6 +40,19 @@ your build dir. After that you just run the build command. Meson will autodetect changes in your source tree and regenerates all files needed to build the project. +The setup command is the default operation. If no actual command is +specified, Meson will assume you meant to do a setup. That means +that you can set up a build directory without the setup command +like this: + +.B meson [ +.I options +.B ] [ +.I source directory +.B ] [ +.I build directory +.B ] + .SS "options:" .TP \fB\-\-version\fR @@ -42,5 +60,146 @@ print version number .TP \fB\-\-help\fR print command line help + +.SH The configure command + +Mesonconf provides a way to configure a Meson +project from the command line. Its usage is simple. + +.B meson configure [ +.I build directory +.B ] [ +.I options to set +.B ] + +If build directory is omitted, the current directory is used instead. + +If no parameters are set, mesonconf will print the value of all build +options to the console. + +To set values, use the \-D command line argument like this. + +.B meson configure \-Dopt1=value1 \-Dopt2=value2 + +.SH The introspect command + +Meson introspect is a command designed to make it simple to integrate with +other tools, such as IDEs. The output of this command is in JSON. + +.B meson introspect [ +.I build directory +.B ] [ +.I option +.B ] + +If build directory is omitted, the current directory is used instead. + +.SS "options:" +.TP +\fB\-\-targets\fR +print all top level targets (executables, libraries, etc) +.TP +\fB\-\-target\-files\fR +print the source files of the given target +.TP +\fB\-\-buildsystem\-files\fR +print all files that make up the build system (meson.build, meson_options.txt etc) +.TP +\fB\-\-tests\fR +print all unit tests +.TP +\fB\-\-help\fR +print command line help + +.SH The test command + +Mesontest is a helper tool for running test suites of projects using Meson. +The default way of running tests is to invoke the default build command: + +\fBninja [\fR \fItest\fR \fB]\fR + +Mesontest provides a richer set of tools for invoking tests. + +.SS "options:" +.TP +\fB\-\-repeat\fR +run tests as many times as specified +.TP +\fB\-\-gdb\fR +run tests under gdb +.TP +\fB\-\-list\fR +list all available tests +.TP +\fB\-\-wrapper\fR +invoke all tests via the given wrapper (e.g. valgrind) +.TP +\fB\-C\fR +Change into the given directory before running tests (must be root of build directory). +.TP +\fB\-\-suite\fR +run tests in this suite +.TP +\fB\-\-no\-suite\fR +do not run tests in this suite +.TP +\fB\-\-no\-stdsplit\fR +do not split stderr and stdout in test logs +.TP +\fB\-\-benchmark\fR +run benchmarks instead of tests +.TP +\fB\-\-logbase\fR +base of file name to use for writing test logs +.TP +\fB\-\-num-processes\fR +how many parallel processes to use to run tests +.TP +\fB\-\-verbose\fR +do not redirect stdout and stderr +.TP +\fB\-t\fR +a multiplier to use for test timeout values (usually something like 100 for Valgrind) +.TP +\fB\-\-setup\fR +use the specified test setup + +.SH The wrap command + +Wraptool is a helper utility to manage source dependencies +using the online wrapdb service. + +.B meson wrap < +.I command +.B > [ +.I options +.B ] + +You should run this command in the top level source directory +of your project. + +.SS "Commands:" +.TP +\fBlist\fR +list all available projects +.TP +\fBsearch\fR +search projects by name +.TP +\fBinstall\fR +install a project with the given name +.TP +\fBupdate\fR +update the specified project to latest available version +.TP +\fBinfo\fR +show available versions of the specified project +.TP +\fBstatus\fR +show installed and available versions of currently used subprojects + .SH SEE ALSO + http://mesonbuild.com/ + +https://wrapdb.mesonbuild.com/ diff --git a/man/mesonconf.1 b/man/mesonconf.1 index 6bb9d39e5..ff9dba1a6 100644 --- a/man/mesonconf.1 +++ b/man/mesonconf.1 @@ -3,23 +3,8 @@ mesonconf - a tool to configure Meson builds .SH DESCRIPTION -Mesonconf provides a way to configure a Meson -project from the command line. Its usage is simple. +This executable is deprecated and will be removed in the future. The +functionality that was in this executable can be invoked via the main Meson +command like this: -.B mesonconf [ -.I build directory -.B ] [ -.I options to set -.B ] - -If build directory is omitted, the current directory is used instead. - -If no parameters are set, mesonconf will print the value of all build -options to the console. - -To set values, just use the \-D command line argument like this. - -.B mesonconf \-Dopt1=value1 \-Dopt2=value2 - -.SH SEE ALSO -http://mesonbuild.com/ +.B meson configure diff --git a/man/mesonintrospect.1 b/man/mesonintrospect.1 index 1918b4f75..78727acba 100644 --- a/man/mesonintrospect.1 +++ b/man/mesonintrospect.1 @@ -3,33 +3,11 @@ mesonintrospect - a tool to extract information about a Meson build .SH DESCRIPTION -Mesonintrospect is a tool designed to make it simple to integrate with -other tools, such as IDEs. The output of this command is in JSON. +This executable is deprecated and will be removed in the future. The +functionality that was in this executable can be invoked via the main Meson +command like this: -.B mesonintrospect [ -.I build directory -.B ] [ -.I option -.B ] - -If build directory is omitted, the current directory is used instead. - -.SS "options:" -.TP -\fB\-\-targets\fR -print all top level targets (executables, libraries, etc) -.TP -\fB\-\-target\-files\fR -print the source files of the given target -.TP -\fB\-\-buildsystem\-files\fR -print all files that make up the build system (meson.build, meson_options.txt etc) -.TP -\fB\-\-tests\fR -print all unit tests -.TP -\fB\-\-help\fR -print command line help +.B meson introspect .SH SEE ALSO http://mesonbuild.com/ diff --git a/man/mesontest.1 b/man/mesontest.1 index cf21fe72b..d806f1936 100644 --- a/man/mesontest.1 +++ b/man/mesontest.1 @@ -3,55 +3,11 @@ mesontest - test tool for the Meson build system .SH DESCRIPTION -Mesontest is a helper tool for running test suites of projects using Meson. -The default way of running tests is to invoke the default build command: +This executable is deprecated and will be removed in the future. The +functionality that was in this executable can be invoked via the main Meson +command like this: -\fBninja [\fR \fItest\fR \fB]\fR +.B meson test -Mesontest provides a much richer set of tools for invoking tests. - -.SS "options:" -.TP -\fB\-\-repeat\fR -run tests as many times as specified -.TP -\fB\-\-gdb\fR -run tests under gdb -.TP -\fB\-\-list\fR -list all available tests -.TP -\fB\-\-wrapper\fR -invoke all tests via the given wrapper (e.g. valgrind) -.TP -\fB\-C\fR -Change into the given directory before running tests (must be root of build directory). -.TP -\fB\-\-suite\fR -run tests in this suite -.TP -\fB\-\-no\-suite\fR -do not run tests in this suite -.TP -\fB\-\-no\-stdsplit\fR -do not split stderr and stdout in test logs -.TP -\fB\-\-benchmark\fR -run benchmarks instead of tests -.TP -\fB\-\-logbase\fR -base of file name to use for writing test logs -.TP -\fB\-\-num-processes\fR -how many parallel processes to use to run tests -.TP -\fB\-\-verbose\fR -do not redirect stdout and stderr -.TP -\fB\-t\fR -a multiplier to use for test timeout values (usually something like 100 for Valgrind) -.TP -\fB\-\-setup\fR -use the specified test setup .SH SEE ALSO http://mesonbuild.com/ diff --git a/man/wraptool.1 b/man/wraptool.1 index 73c25683a..94d79308d 100644 --- a/man/wraptool.1 +++ b/man/wraptool.1 @@ -3,36 +3,11 @@ wraptool - source dependency downloader .SH DESCRIPTION -Wraptool is a helper utility to manage source dependencies -using the wrapdb database. +This executable is deprecated and will be removed in the future. The +functionality that was in this executable can be invoked via the main Meson +command like this: -.B wraptool < -.I command -.B > [ -.I options -.B ] +.B meson wrap -You should run this command in the top level source directory -of your project. - -.SS "Commands:" -.TP -\fBlist\fR -list all available projects -.TP -\fBsearch\fR -search projects by name -.TP -\fBinstall\fR -install a project with the given name -.TP -\fBupdate\fR -update the specified project to latest available version -.TP -\fBinfo\fR -show available versions of the specified project -.TP -\fBstatus\fR -show installed and available versions of currently used subprojects .SH SEE ALSO http://wrapdb.mesonbuild.com/ diff --git a/meson.py b/meson.py index ca29d1876..d1b629d6f 100755 --- a/meson.py +++ b/meson.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mesonbuild import mlog, mesonmain, mesonlib +from mesonbuild import mesonmain, mesonlib import sys, os, locale def main(): @@ -24,14 +24,14 @@ def main(): # encoding, so we can just warn about it. e = locale.getpreferredencoding() if e.upper() != 'UTF-8' and not mesonlib.is_windows(): - mlog.warning('You are using {!r} which is not a a Unicode-compatible ' - 'locale.'.format(e)) - mlog.warning('You might see errors if you use UTF-8 strings as ' - 'filenames, as strings, or as file contents.') - mlog.warning('Please switch to a UTF-8 locale for your platform.') + print('Warning: You are using {!r} which is not a a Unicode-compatible ' + 'locale.'.format(e), file=sys.stderr) + print('You might see errors if you use UTF-8 strings as ' + 'filenames, as strings, or as file contents.', file=sys.stderr) + print('Please switch to a UTF-8 locale for your platform.', file=sys.stderr) # Always resolve the command path so Ninja can find it for regen, tests, etc. launcher = os.path.realpath(sys.argv[0]) - return mesonmain.run(launcher, sys.argv[1:]) + return mesonmain.run(sys.argv[1:], launcher) if __name__ == '__main__': sys.exit(main()) diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index 016494550..ed23f5238 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -17,10 +17,11 @@ import time, datetime import os.path from . import environment, interpreter, mesonlib from . import build +from . import mconf, mintro, mtest, rewriter import platform from . import mlog, coredata from .mesonlib import MesonException -from .wrap import WrapMode +from .wrap import WrapMode, wraptool parser = argparse.ArgumentParser() @@ -262,12 +263,33 @@ def run_script_command(args): raise MesonException('Unknown internal command {}.'.format(cmdname)) return cmdfunc(cmdargs) -def run(mainfile, args): - if sys.version_info < (3, 3): - print('Meson works correctly only with python 3.3+.') +def run(args, mainfile=None): + if sys.version_info < (3, 4): + print('Meson works correctly only with python 3.4+.') print('You have python %s.' % sys.version) print('Please update your environment') return 1 + if len(args) > 0: + # First check if we want to run a subcommand. + cmd_name = args[0] + remaining_args = args[1:] + if cmd_name == 'test': + return mtest.run(remaining_args) + elif cmd_name == 'setup': + args = remaining_args + # FALLTHROUGH like it's 1972. + elif cmd_name == 'introspect': + return mintro.run(remaining_args) + elif cmd_name == 'test': + return mtest.run(remaining_args) + elif cmd_name == 'rewrite': + return rewriter.run(remaining_args) + elif cmd_name == 'configure': + return mconf.run(remaining_args) + elif cmd_name == 'wrap': + return wraptool.run(remaining_args) + + # No special command? Do the basic setup/reconf. if len(args) >= 2 and args[0] == '--internal': if args[1] != 'regenerate': script = args[1] @@ -281,6 +303,7 @@ def run(mainfile, args): handshake = True else: handshake = False + args = mesonlib.expand_arguments(args) options = parser.parse_args(args) args = options.directories @@ -302,6 +325,8 @@ def run(mainfile, args): else: dir2 = '.' try: + if mainfile is None: + sys.exit('I iz broken. Sorry.') app = MesonApp(dir1, dir2, mainfile, handshake, options, sys.argv) except Exception as e: # Log directory does not exist, so just print diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py new file mode 100644 index 000000000..2520ae822 --- /dev/null +++ b/mesonbuild/mtest.py @@ -0,0 +1,624 @@ +# Copyright 2016-2017 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A tool to run tests in many different ways. + +import shlex +import subprocess, sys, os, argparse +import pickle +from mesonbuild import build +from mesonbuild import environment +from mesonbuild.dependencies import ExternalProgram +from mesonbuild import mesonlib +from mesonbuild import mlog + +import time, datetime, multiprocessing, json +import concurrent.futures as conc +import platform +import signal +import random + +# GNU autotools interprets a return code of 77 from tests it executes to +# mean that the test should be skipped. +GNU_SKIP_RETURNCODE = 77 + +def is_windows(): + platname = platform.system().lower() + return platname == 'windows' or 'mingw' in platname + +def is_cygwin(): + platname = platform.system().lower() + return 'cygwin' in platname + +def determine_worker_count(): + varname = 'MESON_TESTTHREADS' + if varname in os.environ: + try: + num_workers = int(os.environ[varname]) + except ValueError: + print('Invalid value in %s, using 1 thread.' % varname) + num_workers = 1 + else: + try: + # Fails in some weird environments such as Debian + # reproducible build. + num_workers = multiprocessing.cpu_count() + except Exception: + num_workers = 1 + return num_workers + +parser = argparse.ArgumentParser() +parser.add_argument('--repeat', default=1, dest='repeat', type=int, + help='Number of times to run the tests.') +parser.add_argument('--no-rebuild', default=False, action='store_true', + help='Do not rebuild before running tests.') +parser.add_argument('--gdb', default=False, dest='gdb', action='store_true', + help='Run test under gdb.') +parser.add_argument('--list', default=False, dest='list', action='store_true', + help='List available tests.') +parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split, + help='wrapper to run tests with (e.g. Valgrind)') +parser.add_argument('-C', default='.', dest='wd', + help='directory to cd into before running') +parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', + help='Only run tests belonging to the given suite.') +parser.add_argument('--no-suite', default=[], dest='exclude_suites', action='append', metavar='SUITE', + help='Do not run tests belonging to the given suite.') +parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', + help='Do not split stderr and stdout in test logs.') +parser.add_argument('--print-errorlogs', default=False, action='store_true', + help="Whether to print failing tests' logs.") +parser.add_argument('--benchmark', default=False, action='store_true', + help="Run benchmarks instead of tests.") +parser.add_argument('--logbase', default='testlog', + help="Base name for log file.") +parser.add_argument('--num-processes', default=determine_worker_count(), type=int, + help='How many parallel processes to use.') +parser.add_argument('-v', '--verbose', default=False, action='store_true', + help='Do not redirect stdout and stderr') +parser.add_argument('-q', '--quiet', default=False, action='store_true', + help='Produce less output to the terminal.') +parser.add_argument('-t', '--timeout-multiplier', type=float, default=None, + help='Define a multiplier for test timeout, for example ' + ' when running tests in particular conditions they might take' + ' more time to execute.') +parser.add_argument('--setup', default=None, dest='setup', + help='Which test setup to use.') +parser.add_argument('--test-args', default=[], type=shlex.split, + help='Arguments to pass to the specified test(s) or all tests') +parser.add_argument('args', nargs='*', + help='Optional list of tests to run') + + +class TestException(mesonlib.MesonException): + pass + + +class TestRun: + def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd, + env): + self.res = res + self.returncode = returncode + self.duration = duration + self.stdo = stdo + self.stde = stde + self.cmd = cmd + self.env = env + self.should_fail = should_fail + + def get_log(self): + res = '--- command ---\n' + if self.cmd is None: + res += 'NONE\n' + else: + res += "%s%s\n" % (''.join(["%s='%s' " % (k, v) for k, v in self.env.items()]), ' ' .join(self.cmd)) + if self.stdo: + res += '--- stdout ---\n' + res += self.stdo + if self.stde: + if res[-1:] != '\n': + res += '\n' + res += '--- stderr ---\n' + res += self.stde + if res[-1:] != '\n': + res += '\n' + res += '-------\n\n' + return res + +def decode(stream): + if stream is None: + return '' + try: + return stream.decode('utf-8') + except UnicodeDecodeError: + return stream.decode('iso-8859-1', errors='ignore') + +def write_json_log(jsonlogfile, test_name, result): + jresult = {'name': test_name, + 'stdout': result.stdo, + 'result': result.res, + 'duration': result.duration, + 'returncode': result.returncode, + 'command': result.cmd} + if isinstance(result.env, dict): + jresult['env'] = result.env + else: + jresult['env'] = result.env.get_env(os.environ) + if result.stde: + jresult['stderr'] = result.stde + jsonlogfile.write(json.dumps(jresult) + '\n') + +def run_with_mono(fname): + if fname.endswith('.exe') and not (is_windows() or is_cygwin()): + return True + return False + +class TestHarness: + def __init__(self, options): + self.options = options + self.collected_logs = [] + self.fail_count = 0 + self.success_count = 0 + self.skip_count = 0 + self.timeout_count = 0 + self.is_run = False + self.tests = None + self.suites = None + self.logfilename = None + self.logfile = None + self.jsonlogfile = None + if self.options.benchmark: + datafile = os.path.join(options.wd, 'meson-private', 'meson_benchmark_setup.dat') + else: + datafile = os.path.join(options.wd, 'meson-private', 'meson_test_setup.dat') + if not os.path.isfile(datafile): + raise TestException('Directory %s does not seem to be a Meson build directory.' % options.wd) + self.load_datafile(datafile) + + def __del__(self): + if self.logfile: + self.logfile.close() + if self.jsonlogfile: + self.jsonlogfile.close() + + def run_single_test(self, wrap, test): + if test.fname[0].endswith('.jar'): + cmd = ['java', '-jar'] + test.fname + elif not test.is_cross and run_with_mono(test.fname[0]): + cmd = ['mono'] + test.fname + else: + if test.is_cross: + if test.exe_runner is None: + # Can not run test on cross compiled executable + # because there is no execute wrapper. + cmd = None + else: + cmd = [test.exe_runner] + test.fname + else: + cmd = test.fname + + if cmd is None: + res = 'SKIP' + duration = 0.0 + stdo = 'Not run because can not execute cross compiled binaries.' + stde = None + returncode = GNU_SKIP_RETURNCODE + else: + cmd = wrap + cmd + test.cmd_args + self.options.test_args + starttime = time.time() + child_env = os.environ.copy() + child_env.update(self.options.global_env.get_env(child_env)) + if isinstance(test.env, build.EnvironmentVariables): + test.env = test.env.get_env(child_env) + + child_env.update(test.env) + if len(test.extra_paths) > 0: + child_env['PATH'] += os.pathsep.join([''] + test.extra_paths) + + # If MALLOC_PERTURB_ is not set, or if it is set to an empty value, + # (i.e., the test or the environment don't explicitly set it), set + # it ourselves. We do this unconditionally because it is extremely + # useful to have in tests. + # Setting MALLOC_PERTURB_="0" will completely disable this feature. + if 'MALLOC_PERTURB_' not in child_env or not child_env['MALLOC_PERTURB_']: + child_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) + + setsid = None + stdout = None + stderr = None + if not self.options.verbose: + stdout = subprocess.PIPE + stderr = subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT + + if not is_windows(): + setsid = os.setsid + + p = subprocess.Popen(cmd, + stdout=stdout, + stderr=stderr, + env=child_env, + cwd=test.workdir, + preexec_fn=setsid) + timed_out = False + if test.timeout is None: + timeout = None + else: + timeout = test.timeout * self.options.timeout_multiplier + try: + (stdo, stde) = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + if self.options.verbose: + print("%s time out (After %d seconds)" % (test.name, timeout)) + timed_out = True + # Python does not provide multiplatform support for + # killing a process and all its children so we need + # to roll our own. + if is_windows(): + subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)]) + else: + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + (stdo, stde) = p.communicate() + endtime = time.time() + duration = endtime - starttime + stdo = decode(stdo) + if stde: + stde = decode(stde) + if timed_out: + res = 'TIMEOUT' + self.timeout_count += 1 + self.fail_count += 1 + elif p.returncode == GNU_SKIP_RETURNCODE: + res = 'SKIP' + self.skip_count += 1 + elif test.should_fail == bool(p.returncode): + res = 'OK' + self.success_count += 1 + else: + res = 'FAIL' + self.fail_count += 1 + returncode = p.returncode + result = TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env) + + return result + + def print_stats(self, numlen, tests, name, result, i): + startpad = ' ' * (numlen - len('%d' % (i + 1))) + num = '%s%d/%d' % (startpad, i + 1, len(tests)) + padding1 = ' ' * (38 - len(name)) + padding2 = ' ' * (8 - len(result.res)) + result_str = '%s %s %s%s%s%5.2f s' % \ + (num, name, padding1, result.res, padding2, result.duration) + if not self.options.quiet or result.res != 'OK': + if result.res != 'OK' and mlog.colorize_console: + if result.res == 'FAIL' or result.res == 'TIMEOUT': + decorator = mlog.red + elif result.res == 'SKIP': + decorator = mlog.yellow + else: + sys.exit('Unreachable code was ... well ... reached.') + print(decorator(result_str).get_text(True)) + else: + print(result_str) + result_str += "\n\n" + result.get_log() + if (result.returncode != GNU_SKIP_RETURNCODE) \ + and (result.returncode != 0) != result.should_fail: + if self.options.print_errorlogs: + self.collected_logs.append(result_str) + if self.logfile: + self.logfile.write(result_str) + if self.jsonlogfile: + write_json_log(self.jsonlogfile, name, result) + + def print_summary(self): + msg = ''' +OK: %4d +FAIL: %4d +SKIP: %4d +TIMEOUT: %4d +''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count) + print(msg) + if self.logfile: + self.logfile.write(msg) + + def print_collected_logs(self): + if len(self.collected_logs) > 0: + if len(self.collected_logs) > 10: + print('\nThe output from 10 first failed tests:\n') + else: + print('\nThe output from the failed tests:\n') + for log in self.collected_logs[:10]: + lines = log.splitlines() + if len(lines) > 104: + print('\n'.join(lines[0:4])) + print('--- Listing only the last 100 lines from a long log. ---') + lines = lines[-100:] + for line in lines: + print(line) + + def doit(self): + if self.is_run: + raise RuntimeError('Test harness object can only be used once.') + if not os.path.isfile(self.datafile): + print('Test data file. Probably this means that you did not run this in the build directory.') + return 1 + self.is_run = True + tests = self.get_tests() + if not tests: + return 0 + self.run_tests(tests) + return self.fail_count + + @staticmethod + def split_suite_string(suite): + if ':' in suite: + return suite.split(':', 1) + else: + return suite, "" + + @staticmethod + def test_in_suites(test, suites): + for suite in suites: + (prj_match, st_match) = TestHarness.split_suite_string(suite) + for prjst in test.suite: + (prj, st) = TestHarness.split_suite_string(prjst) + if prj_match and prj != prj_match: + continue + if st_match and st != st_match: + continue + return True + return False + + def test_suitable(self, test): + return (not self.options.include_suites or TestHarness.test_in_suites(test, self.options.include_suites)) \ + and not TestHarness.test_in_suites(test, self.options.exclude_suites) + + def load_suites(self): + ss = set() + for t in self.tests: + for s in t.suite: + ss.add(s) + self.suites = list(ss) + + def load_tests(self): + with open(self.datafile, 'rb') as f: + self.tests = pickle.load(f) + + def load_datafile(self, datafile): + self.datafile = datafile + self.load_tests() + self.load_suites() + + def get_tests(self): + if not self.tests: + print('No tests defined.') + return [] + + if len(self.options.include_suites) or len(self.options.exclude_suites): + tests = [] + for tst in self.tests: + if self.test_suitable(tst): + tests.append(tst) + else: + tests = self.tests + + if self.options.args: + tests = [t for t in tests if t.name in self.options.args] + + if not tests: + print('No suitable tests defined.') + return [] + + for test in tests: + test.rebuilt = False + + return tests + + def open_log_files(self): + if not self.options.logbase or self.options.verbose: + return None, None, None, None + + namebase = None + logfile_base = os.path.join(self.options.wd, 'meson-logs', self.options.logbase) + + if self.options.wrapper: + namebase = os.path.split(self.get_wrapper()[0])[1] + elif self.options.setup: + namebase = self.options.setup + + if namebase: + logfile_base += '-' + namebase.replace(' ', '_') + self.logfilename = logfile_base + '.txt' + self.jsonlogfilename = logfile_base + '.json' + + self.jsonlogfile = open(self.jsonlogfilename, 'w') + self.logfile = open(self.logfilename, 'w') + + self.logfile.write('Log of Meson test suite run on %s\n\n' + % datetime.datetime.now().isoformat()) + + def get_wrapper(self): + wrap = [] + if self.options.gdb: + wrap = ['gdb', '--quiet', '--nh'] + if self.options.repeat > 1: + wrap += ['-ex', 'run', '-ex', 'quit'] + # Signal the end of arguments to gdb + wrap += ['--args'] + if self.options.wrapper: + wrap += self.options.wrapper + assert(isinstance(wrap, list)) + return wrap + + def get_pretty_suite(self, test): + if len(self.suites) > 1: + rv = TestHarness.split_suite_string(test.suite[0])[0] + s = "+".join(TestHarness.split_suite_string(s)[1] for s in test.suite) + if len(s): + rv += ":" + return rv + s + " / " + test.name + else: + return test.name + + def run_tests(self, tests): + executor = None + futures = [] + numlen = len('%d' % len(tests)) + self.open_log_files() + wrap = self.get_wrapper() + + for _ in range(self.options.repeat): + for i, test in enumerate(tests): + visible_name = self.get_pretty_suite(test) + + if self.options.gdb: + test.timeout = None + + if not test.is_parallel or self.options.gdb: + self.drain_futures(futures) + futures = [] + res = self.run_single_test(wrap, test) + self.print_stats(numlen, tests, visible_name, res, i) + else: + if not executor: + executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes) + f = executor.submit(self.run_single_test, wrap, test) + futures.append((f, numlen, tests, visible_name, i)) + if self.options.repeat > 1 and self.fail_count: + break + if self.options.repeat > 1 and self.fail_count: + break + + self.drain_futures(futures) + self.print_summary() + self.print_collected_logs() + + if self.logfilename: + print('Full log written to %s' % self.logfilename) + + def drain_futures(self, futures): + for i in futures: + (result, numlen, tests, name, i) = i + if self.options.repeat > 1 and self.fail_count: + result.cancel() + if self.options.verbose: + result.result() + self.print_stats(numlen, tests, name, result.result(), i) + + def run_special(self): + 'Tests run by the user, usually something like "under gdb 1000 times".' + if self.is_run: + raise RuntimeError('Can not use run_special after a full run.') + tests = self.get_tests() + if not tests: + return 0 + self.run_tests(tests) + return self.fail_count + + +def list_tests(th): + tests = th.get_tests() + for t in tests: + print(th.get_pretty_suite(t)) + +def merge_suite_options(options): + buildfile = os.path.join(options.wd, 'meson-private/build.dat') + with open(buildfile, 'rb') as f: + build = pickle.load(f) + setups = build.test_setups + if options.setup not in setups: + sys.exit('Unknown test setup: %s' % options.setup) + current = setups[options.setup] + if not options.gdb: + options.gdb = current.gdb + if options.timeout_multiplier is None: + options.timeout_multiplier = current.timeout_multiplier +# if options.env is None: +# options.env = current.env # FIXME, should probably merge options here. + if options.wrapper is not None and current.exe_wrapper is not None: + sys.exit('Conflict: both test setup and command line specify an exe wrapper.') + if options.wrapper is None: + options.wrapper = current.exe_wrapper + return current.env + +def rebuild_all(wd): + if not os.path.isfile(os.path.join(wd, 'build.ninja')): + print("Only ninja backend is supported to rebuild tests before running them.") + return True + + ninja = environment.detect_ninja() + if not ninja: + print("Can't find ninja, can't rebuild test.") + return False + + p = subprocess.Popen([ninja, '-C', wd]) + p.communicate() + + if p.returncode != 0: + print("Could not rebuild") + return False + + return True + +def run(args): + options = parser.parse_args(args) + + if options.benchmark: + options.num_processes = 1 + + if options.setup is not None: + global_env = merge_suite_options(options) + else: + global_env = build.EnvironmentVariables() + if options.timeout_multiplier is None: + options.timeout_multiplier = 1 + + setattr(options, 'global_env', global_env) + + if options.verbose and options.quiet: + print('Can not be both quiet and verbose at the same time.') + return 1 + + check_bin = None + if options.gdb: + options.verbose = True + if options.wrapper: + print('Must not specify both a wrapper and gdb at the same time.') + return 1 + check_bin = 'gdb' + + if options.wrapper: + check_bin = options.wrapper[0] + + if check_bin is not None: + exe = ExternalProgram(check_bin, silent=True) + if not exe.found(): + sys.exit("Could not find requested program: %s" % check_bin) + options.wd = os.path.abspath(options.wd) + + if not options.list and not options.no_rebuild: + if not rebuild_all(options.wd): + sys.exit(-1) + + try: + th = TestHarness(options) + if options.list: + list_tests(th) + return 0 + if not options.args: + return th.doit() + return th.run_special() + except TestException as e: + print('Mesontest encountered an error:\n') + print(e) + return 1 diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py new file mode 100755 index 000000000..b88c5efe2 --- /dev/null +++ b/mesonbuild/rewriter.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# Copyright 2016 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +# This tool is used to manipulate an existing Meson build definition. +# +# - add a file to a target +# - remove files from a target +# - move targets +# - reindent? + +import mesonbuild.astinterpreter +from mesonbuild.mesonlib import MesonException +from mesonbuild import mlog +import sys, traceback +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument('--sourcedir', default='.', + help='Path to source directory.') +parser.add_argument('--target', default=None, + help='Name of target to edit.') +parser.add_argument('--filename', default=None, + help='Name of source file to add or remove to target.') +parser.add_argument('commands', nargs='+') + +def run(args): + options = parser.parse_args(args) + if options.target is None or options.filename is None: + sys.exit("Must specify both target and filename.") + print('This tool is highly experimental, use with care.') + rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '') + try: + if options.commands[0] == 'add': + rewriter.add_source(options.target, options.filename) + elif options.commands[0] == 'remove': + rewriter.remove_source(options.target, options.filename) + else: + sys.exit('Unknown command: ' + options.commands[0]) + except Exception as e: + if isinstance(e, MesonException): + if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(e, 'colno'): + mlog.log(mlog.red('\nMeson encountered an error in file %s, line %d, column %d:' % (e.file, e.lineno, e.colno))) + else: + mlog.log(mlog.red('\nMeson encountered an error:')) + mlog.log(e) + else: + traceback.print_exc() + return 1 + return 0 diff --git a/mesonconf.py b/mesonconf.py index 2b0a1a61f..d1874e0e3 100755 --- a/mesonconf.py +++ b/mesonconf.py @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mesonbuild import mconf +from mesonbuild import mesonmain import sys -sys.exit(mconf.run(sys.argv[1:])) +if __name__ == '__main__': + print('Warning: This executable is deprecated. Use "meson configure" instead.', + file=sys.stderr) + sys.exit(mesonmain.run(['configure'] + sys.argv[1:])) diff --git a/mesonintrospect.py b/mesonintrospect.py index 4d20548b1..5cc07bfb7 100755 --- a/mesonintrospect.py +++ b/mesonintrospect.py @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mesonbuild import mintro +from mesonbuild import mesonmain import sys -sys.exit(mintro.run(sys.argv[1:])) +if __name__ == '__main__': + print('Warning: This executable is deprecated. Use "meson introspect" instead.', + file=sys.stderr) + sys.exit(mesonmain.run(['introspect'] + sys.argv[1:])) diff --git a/mesonrewriter.py b/mesonrewriter.py index fb857458c..426d87848 100755 --- a/mesonrewriter.py +++ b/mesonrewriter.py @@ -23,42 +23,11 @@ # - move targets # - reindent? -import mesonbuild.astinterpreter -from mesonbuild.mesonlib import MesonException -from mesonbuild import mlog -import sys, traceback -import argparse - -parser = argparse.ArgumentParser() - -parser.add_argument('--sourcedir', default='.', - help='Path to source directory.') -parser.add_argument('--target', default=None, - help='Name of target to edit.') -parser.add_argument('--filename', default=None, - help='Name of source file to add or remove to target.') -parser.add_argument('commands', nargs='+') +from mesonbuild import mesonmain, mlog +import sys if __name__ == '__main__': - options = parser.parse_args() - if options.target is None or options.filename is None: - sys.exit("Must specify both target and filename.") - print('This tool is highly experimental, use with care.') - rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '') - try: - if options.commands[0] == 'add': - rewriter.add_source(options.target, options.filename) - elif options.commands[0] == 'remove': - rewriter.remove_source(options.target, options.filename) - else: - sys.exit('Unknown command: ' + options.commands[0]) - except Exception as e: - if isinstance(e, MesonException): - if hasattr(e, 'file') and hasattr(e, 'lineno') and hasattr(e, 'colno'): - mlog.log(mlog.red('\nMeson encountered an error in file %s, line %d, column %d:' % (e.file, e.lineno, e.colno))) - else: - mlog.log(mlog.red('\nMeson encountered an error:')) - mlog.log(e) - else: - traceback.print_exc() - sys.exit(1) + print('Warning: This executable is deprecated. Use "meson rewrite" instead.', + file=sys.stderr) + sys.exit(mesonmain.run(['rewrite'] + sys.argv[1:])) + diff --git a/mesontest.py b/mesontest.py index 83044aa64..c2d39d69a 100755 --- a/mesontest.py +++ b/mesontest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2016 The Meson development team +# Copyright 2016-2017 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,614 +16,10 @@ # A tool to run tests in many different ways. -import shlex -import subprocess, sys, os, argparse -import pickle -from mesonbuild import build -from mesonbuild import environment -from mesonbuild.dependencies import ExternalProgram -from mesonbuild import mesonlib -from mesonbuild import mlog - -import time, datetime, multiprocessing, json -import concurrent.futures as conc -import platform -import signal -import random - -# GNU autotools interprets a return code of 77 from tests it executes to -# mean that the test should be skipped. -GNU_SKIP_RETURNCODE = 77 - -def is_windows(): - platname = platform.system().lower() - return platname == 'windows' or 'mingw' in platname - -def is_cygwin(): - platname = platform.system().lower() - return 'cygwin' in platname - -def determine_worker_count(): - varname = 'MESON_TESTTHREADS' - if varname in os.environ: - try: - num_workers = int(os.environ[varname]) - except ValueError: - print('Invalid value in %s, using 1 thread.' % varname) - num_workers = 1 - else: - try: - # Fails in some weird environments such as Debian - # reproducible build. - num_workers = multiprocessing.cpu_count() - except Exception: - num_workers = 1 - return num_workers - -parser = argparse.ArgumentParser() -parser.add_argument('--repeat', default=1, dest='repeat', type=int, - help='Number of times to run the tests.') -parser.add_argument('--no-rebuild', default=False, action='store_true', - help='Do not rebuild before running tests.') -parser.add_argument('--gdb', default=False, dest='gdb', action='store_true', - help='Run test under gdb.') -parser.add_argument('--list', default=False, dest='list', action='store_true', - help='List available tests.') -parser.add_argument('--wrapper', default=None, dest='wrapper', type=shlex.split, - help='wrapper to run tests with (e.g. Valgrind)') -parser.add_argument('-C', default='.', dest='wd', - help='directory to cd into before running') -parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE', - help='Only run tests belonging to the given suite.') -parser.add_argument('--no-suite', default=[], dest='exclude_suites', action='append', metavar='SUITE', - help='Do not run tests belonging to the given suite.') -parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false', - help='Do not split stderr and stdout in test logs.') -parser.add_argument('--print-errorlogs', default=False, action='store_true', - help="Whether to print failing tests' logs.") -parser.add_argument('--benchmark', default=False, action='store_true', - help="Run benchmarks instead of tests.") -parser.add_argument('--logbase', default='testlog', - help="Base name for log file.") -parser.add_argument('--num-processes', default=determine_worker_count(), type=int, - help='How many parallel processes to use.') -parser.add_argument('-v', '--verbose', default=False, action='store_true', - help='Do not redirect stdout and stderr') -parser.add_argument('-q', '--quiet', default=False, action='store_true', - help='Produce less output to the terminal.') -parser.add_argument('-t', '--timeout-multiplier', type=float, default=None, - help='Define a multiplier for test timeout, for example ' - ' when running tests in particular conditions they might take' - ' more time to execute.') -parser.add_argument('--setup', default=None, dest='setup', - help='Which test setup to use.') -parser.add_argument('--test-args', default=[], type=shlex.split, - help='Arguments to pass to the specified test(s) or all tests') -parser.add_argument('args', nargs='*', - help='Optional list of tests to run') - - -class TestException(mesonlib.MesonException): - pass - - -class TestRun: - def __init__(self, res, returncode, should_fail, duration, stdo, stde, cmd, - env): - self.res = res - self.returncode = returncode - self.duration = duration - self.stdo = stdo - self.stde = stde - self.cmd = cmd - self.env = env - self.should_fail = should_fail - - def get_log(self): - res = '--- command ---\n' - if self.cmd is None: - res += 'NONE\n' - else: - res += "%s%s\n" % (''.join(["%s='%s' " % (k, v) for k, v in self.env.items()]), ' ' .join(self.cmd)) - if self.stdo: - res += '--- stdout ---\n' - res += self.stdo - if self.stde: - if res[-1:] != '\n': - res += '\n' - res += '--- stderr ---\n' - res += self.stde - if res[-1:] != '\n': - res += '\n' - res += '-------\n\n' - return res - -def decode(stream): - if stream is None: - return '' - try: - return stream.decode('utf-8') - except UnicodeDecodeError: - return stream.decode('iso-8859-1', errors='ignore') - -def write_json_log(jsonlogfile, test_name, result): - jresult = {'name': test_name, - 'stdout': result.stdo, - 'result': result.res, - 'duration': result.duration, - 'returncode': result.returncode, - 'command': result.cmd} - if isinstance(result.env, dict): - jresult['env'] = result.env - else: - jresult['env'] = result.env.get_env(os.environ) - if result.stde: - jresult['stderr'] = result.stde - jsonlogfile.write(json.dumps(jresult) + '\n') - -def run_with_mono(fname): - if fname.endswith('.exe') and not (is_windows() or is_cygwin()): - return True - return False - -class TestHarness: - def __init__(self, options): - self.options = options - self.collected_logs = [] - self.fail_count = 0 - self.success_count = 0 - self.skip_count = 0 - self.timeout_count = 0 - self.is_run = False - self.tests = None - self.suites = None - self.logfilename = None - self.logfile = None - self.jsonlogfile = None - if self.options.benchmark: - datafile = os.path.join(options.wd, 'meson-private', 'meson_benchmark_setup.dat') - else: - datafile = os.path.join(options.wd, 'meson-private', 'meson_test_setup.dat') - if not os.path.isfile(datafile): - raise TestException('Directory %s does not seem to be a Meson build directory.' % options.wd) - self.load_datafile(datafile) - - def __del__(self): - if self.logfile: - self.logfile.close() - if self.jsonlogfile: - self.jsonlogfile.close() - - def run_single_test(self, wrap, test): - if test.fname[0].endswith('.jar'): - cmd = ['java', '-jar'] + test.fname - elif not test.is_cross and run_with_mono(test.fname[0]): - cmd = ['mono'] + test.fname - else: - if test.is_cross: - if test.exe_runner is None: - # Can not run test on cross compiled executable - # because there is no execute wrapper. - cmd = None - else: - cmd = [test.exe_runner] + test.fname - else: - cmd = test.fname - - if cmd is None: - res = 'SKIP' - duration = 0.0 - stdo = 'Not run because can not execute cross compiled binaries.' - stde = None - returncode = GNU_SKIP_RETURNCODE - else: - cmd = wrap + cmd + test.cmd_args + self.options.test_args - starttime = time.time() - child_env = os.environ.copy() - child_env.update(self.options.global_env.get_env(child_env)) - if isinstance(test.env, build.EnvironmentVariables): - test.env = test.env.get_env(child_env) - - child_env.update(test.env) - if len(test.extra_paths) > 0: - child_env['PATH'] += os.pathsep.join([''] + test.extra_paths) - - # If MALLOC_PERTURB_ is not set, or if it is set to an empty value, - # (i.e., the test or the environment don't explicitly set it), set - # it ourselves. We do this unconditionally because it is extremely - # useful to have in tests. - # Setting MALLOC_PERTURB_="0" will completely disable this feature. - if 'MALLOC_PERTURB_' not in child_env or not child_env['MALLOC_PERTURB_']: - child_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) - - setsid = None - stdout = None - stderr = None - if not self.options.verbose: - stdout = subprocess.PIPE - stderr = subprocess.PIPE if self.options and self.options.split else subprocess.STDOUT - - if not is_windows(): - setsid = os.setsid - - p = subprocess.Popen(cmd, - stdout=stdout, - stderr=stderr, - env=child_env, - cwd=test.workdir, - preexec_fn=setsid) - timed_out = False - if test.timeout is None: - timeout = None - else: - timeout = test.timeout * self.options.timeout_multiplier - try: - (stdo, stde) = p.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - if self.options.verbose: - print("%s time out (After %d seconds)" % (test.name, timeout)) - timed_out = True - # Python does not provide multiplatform support for - # killing a process and all its children so we need - # to roll our own. - if is_windows(): - subprocess.call(['taskkill', '/F', '/T', '/PID', str(p.pid)]) - else: - os.killpg(os.getpgid(p.pid), signal.SIGKILL) - (stdo, stde) = p.communicate() - endtime = time.time() - duration = endtime - starttime - stdo = decode(stdo) - if stde: - stde = decode(stde) - if timed_out: - res = 'TIMEOUT' - self.timeout_count += 1 - self.fail_count += 1 - elif p.returncode == GNU_SKIP_RETURNCODE: - res = 'SKIP' - self.skip_count += 1 - elif test.should_fail == bool(p.returncode): - res = 'OK' - self.success_count += 1 - else: - res = 'FAIL' - self.fail_count += 1 - returncode = p.returncode - result = TestRun(res, returncode, test.should_fail, duration, stdo, stde, cmd, test.env) - - return result - - def print_stats(self, numlen, tests, name, result, i): - startpad = ' ' * (numlen - len('%d' % (i + 1))) - num = '%s%d/%d' % (startpad, i + 1, len(tests)) - padding1 = ' ' * (38 - len(name)) - padding2 = ' ' * (8 - len(result.res)) - result_str = '%s %s %s%s%s%5.2f s' % \ - (num, name, padding1, result.res, padding2, result.duration) - if not self.options.quiet or result.res != 'OK': - if result.res != 'OK' and mlog.colorize_console: - if result.res == 'FAIL' or result.res == 'TIMEOUT': - decorator = mlog.red - elif result.res == 'SKIP': - decorator = mlog.yellow - else: - sys.exit('Unreachable code was ... well ... reached.') - print(decorator(result_str).get_text(True)) - else: - print(result_str) - result_str += "\n\n" + result.get_log() - if (result.returncode != GNU_SKIP_RETURNCODE) \ - and (result.returncode != 0) != result.should_fail: - if self.options.print_errorlogs: - self.collected_logs.append(result_str) - if self.logfile: - self.logfile.write(result_str) - if self.jsonlogfile: - write_json_log(self.jsonlogfile, name, result) - - def print_summary(self): - msg = ''' -OK: %4d -FAIL: %4d -SKIP: %4d -TIMEOUT: %4d -''' % (self.success_count, self.fail_count, self.skip_count, self.timeout_count) - print(msg) - if self.logfile: - self.logfile.write(msg) - - def print_collected_logs(self): - if len(self.collected_logs) > 0: - if len(self.collected_logs) > 10: - print('\nThe output from 10 first failed tests:\n') - else: - print('\nThe output from the failed tests:\n') - for log in self.collected_logs[:10]: - lines = log.splitlines() - if len(lines) > 104: - print('\n'.join(lines[0:4])) - print('--- Listing only the last 100 lines from a long log. ---') - lines = lines[-100:] - for line in lines: - print(line) - - def doit(self): - if self.is_run: - raise RuntimeError('Test harness object can only be used once.') - if not os.path.isfile(self.datafile): - print('Test data file. Probably this means that you did not run this in the build directory.') - return 1 - self.is_run = True - tests = self.get_tests() - if not tests: - return 0 - self.run_tests(tests) - return self.fail_count - - @staticmethod - def split_suite_string(suite): - if ':' in suite: - return suite.split(':', 1) - else: - return suite, "" - - @staticmethod - def test_in_suites(test, suites): - for suite in suites: - (prj_match, st_match) = TestHarness.split_suite_string(suite) - for prjst in test.suite: - (prj, st) = TestHarness.split_suite_string(prjst) - if prj_match and prj != prj_match: - continue - if st_match and st != st_match: - continue - return True - return False - - def test_suitable(self, test): - return (not self.options.include_suites or TestHarness.test_in_suites(test, self.options.include_suites)) \ - and not TestHarness.test_in_suites(test, self.options.exclude_suites) - - def load_suites(self): - ss = set() - for t in self.tests: - for s in t.suite: - ss.add(s) - self.suites = list(ss) - - def load_tests(self): - with open(self.datafile, 'rb') as f: - self.tests = pickle.load(f) - - def load_datafile(self, datafile): - self.datafile = datafile - self.load_tests() - self.load_suites() - - def get_tests(self): - if not self.tests: - print('No tests defined.') - return [] - - if len(self.options.include_suites) or len(self.options.exclude_suites): - tests = [] - for tst in self.tests: - if self.test_suitable(tst): - tests.append(tst) - else: - tests = self.tests - - if self.options.args: - tests = [t for t in tests if t.name in self.options.args] - - if not tests: - print('No suitable tests defined.') - return [] - - for test in tests: - test.rebuilt = False - - return tests - - def open_log_files(self): - if not self.options.logbase or self.options.verbose: - return None, None, None, None - - namebase = None - logfile_base = os.path.join(self.options.wd, 'meson-logs', self.options.logbase) - - if self.options.wrapper: - namebase = os.path.split(self.get_wrapper()[0])[1] - elif self.options.setup: - namebase = self.options.setup - - if namebase: - logfile_base += '-' + namebase.replace(' ', '_') - self.logfilename = logfile_base + '.txt' - self.jsonlogfilename = logfile_base + '.json' - - self.jsonlogfile = open(self.jsonlogfilename, 'w') - self.logfile = open(self.logfilename, 'w') - - self.logfile.write('Log of Meson test suite run on %s\n\n' - % datetime.datetime.now().isoformat()) - - def get_wrapper(self): - wrap = [] - if self.options.gdb: - wrap = ['gdb', '--quiet', '--nh'] - if self.options.repeat > 1: - wrap += ['-ex', 'run', '-ex', 'quit'] - # Signal the end of arguments to gdb - wrap += ['--args'] - if self.options.wrapper: - wrap += self.options.wrapper - assert(isinstance(wrap, list)) - return wrap - - def get_pretty_suite(self, test): - if len(self.suites) > 1: - rv = TestHarness.split_suite_string(test.suite[0])[0] - s = "+".join(TestHarness.split_suite_string(s)[1] for s in test.suite) - if len(s): - rv += ":" - return rv + s + " / " + test.name - else: - return test.name - - def run_tests(self, tests): - executor = None - futures = [] - numlen = len('%d' % len(tests)) - self.open_log_files() - wrap = self.get_wrapper() - - for _ in range(self.options.repeat): - for i, test in enumerate(tests): - visible_name = self.get_pretty_suite(test) - - if self.options.gdb: - test.timeout = None - - if not test.is_parallel or self.options.gdb: - self.drain_futures(futures) - futures = [] - res = self.run_single_test(wrap, test) - self.print_stats(numlen, tests, visible_name, res, i) - else: - if not executor: - executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes) - f = executor.submit(self.run_single_test, wrap, test) - futures.append((f, numlen, tests, visible_name, i)) - if self.options.repeat > 1 and self.fail_count: - break - if self.options.repeat > 1 and self.fail_count: - break - - self.drain_futures(futures) - self.print_summary() - self.print_collected_logs() - - if self.logfilename: - print('Full log written to %s' % self.logfilename) - - def drain_futures(self, futures): - for i in futures: - (result, numlen, tests, name, i) = i - if self.options.repeat > 1 and self.fail_count: - result.cancel() - if self.options.verbose: - result.result() - self.print_stats(numlen, tests, name, result.result(), i) - - def run_special(self): - 'Tests run by the user, usually something like "under gdb 1000 times".' - if self.is_run: - raise RuntimeError('Can not use run_special after a full run.') - tests = self.get_tests() - if not tests: - return 0 - self.run_tests(tests) - return self.fail_count - - -def list_tests(th): - tests = th.get_tests() - for t in tests: - print(th.get_pretty_suite(t)) - -def merge_suite_options(options): - buildfile = os.path.join(options.wd, 'meson-private/build.dat') - with open(buildfile, 'rb') as f: - build = pickle.load(f) - setups = build.test_setups - if options.setup not in setups: - sys.exit('Unknown test setup: %s' % options.setup) - current = setups[options.setup] - if not options.gdb: - options.gdb = current.gdb - if options.timeout_multiplier is None: - options.timeout_multiplier = current.timeout_multiplier -# if options.env is None: -# options.env = current.env # FIXME, should probably merge options here. - if options.wrapper is not None and current.exe_wrapper is not None: - sys.exit('Conflict: both test setup and command line specify an exe wrapper.') - if options.wrapper is None: - options.wrapper = current.exe_wrapper - return current.env - -def rebuild_all(wd): - if not os.path.isfile(os.path.join(wd, 'build.ninja')): - print("Only ninja backend is supported to rebuild tests before running them.") - return True - - ninja = environment.detect_ninja() - if not ninja: - print("Can't find ninja, can't rebuild test.") - return False - - p = subprocess.Popen([ninja, '-C', wd]) - p.communicate() - - if p.returncode != 0: - print("Could not rebuild") - return False - - return True - -def run(args): - options = parser.parse_args(args) - - if options.benchmark: - options.num_processes = 1 - - if options.setup is not None: - global_env = merge_suite_options(options) - else: - global_env = build.EnvironmentVariables() - if options.timeout_multiplier is None: - options.timeout_multiplier = 1 - - setattr(options, 'global_env', global_env) - - if options.verbose and options.quiet: - print('Can not be both quiet and verbose at the same time.') - return 1 - - check_bin = None - if options.gdb: - options.verbose = True - if options.wrapper: - print('Must not specify both a wrapper and gdb at the same time.') - return 1 - check_bin = 'gdb' - - if options.wrapper: - check_bin = options.wrapper[0] - - if check_bin is not None: - exe = ExternalProgram(check_bin, silent=True) - if not exe.found(): - sys.exit("Could not find requested program: %s" % check_bin) - options.wd = os.path.abspath(options.wd) - - if not options.list and not options.no_rebuild: - if not rebuild_all(options.wd): - sys.exit(-1) - - try: - th = TestHarness(options) - if options.list: - list_tests(th) - return 0 - if not options.args: - return th.doit() - return th.run_special() - except TestException as e: - print('Mesontest encountered an error:\n') - print(e) - return 1 +from mesonbuild import mesonmain +import sys if __name__ == '__main__': - sys.exit(run(sys.argv[1:])) + print('Warning: This executable is deprecated. Use "meson test" instead.', + file=sys.stderr) + sys.exit(mesonmain.run(['test'] + sys.argv[1:])) diff --git a/msi/License.rtf b/msi/License.rtf new file mode 100755 index 000000000..9b58df930 --- /dev/null +++ b/msi/License.rtf @@ -0,0 +1,73 @@ +{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fprq2\fcharset0 Arial;}} +{\colortbl ;\red0\green0\blue255;} +{\*\generator Msftedit 5.41.21.2510;}\viewkind4\uc1\pard\qc\lang1033\b\f0\fs18 Apache License\par +Version 2.0, January 2004\par +{\field{\*\fldinst{HYPERLINK "http://www.apache.org/licenses/"}}{\fldrslt{\ul\cf1 http://www.apache.org/licenses/}}}\f0\fs18\par +\b0\par +\pard TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\par +\par +\pard\fi-180\li180 1. Definitions.\par +\par +\pard\li180 "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.\par +\par +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.\par +\par +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.\par +\par +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.\par +\par +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.\par +\par +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.\par +\par +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).\par +\par +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.\par +\par +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."\par +\par +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.\par +\pard\par +\pard\fi-180\li180 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.\par +\par +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.\par +\par +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:\par +\pard\par +\pard\fi-270\li450 (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and\par +\par +(b) You must cause any modified files to carry prominent notices stating that You changed the files; and\par +\par +(c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and\par +\par +(d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.\par +\pard\par +\pard\li180 You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.\par +\pard\par +\pard\fi-180\li180 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.\par +\par +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.\par +\par +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.\par +\par +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.\par +\par +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.\par +\pard\par +END OF TERMS AND CONDITIONS\par +\par +APPENDIX: How to apply the Apache License to your work.\par +\par +To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.\par +\par +\pard\li180 Copyright [yyyy] [name of copyright owner]\par +\par +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\par +\par +\pard\li360{\field{\*\fldinst{HYPERLINK "http://www.apache.org/licenses/LICENSE-2.0"}}{\fldrslt{\ul\cf1 http://www.apache.org/licenses/LICENSE-2.0}}}\f0\fs18\par +\pard\li180\par +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\par +\pard\par +\par +} + \ No newline at end of file diff --git a/msi/createmsi.py b/msi/createmsi.py new file mode 100755 index 000000000..4fbc2d6c7 --- /dev/null +++ b/msi/createmsi.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +# Copyright 2017 The Meson development team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, os, subprocess, shutil, uuid +from glob import glob +import platform + +sys.path.append(os.getcwd()) +from mesonbuild import coredata + +xml_templ = ''' + + + + + + + + + +''' + +xml_footer_templ = ''' + + + + + +%s + + + + + + + +''' + +file_templ = ''' +''' + +comp_ref_templ = ''' +''' + +path_addition_xml = ''' +''' + + +def gen_guid(): + return str(uuid.uuid4()).upper() + +class Node: + def __init__(self, dirs, files): + assert(isinstance(dirs, list)) + assert(isinstance(files, list)) + self.dirs = dirs + self.files = files + +class PackageGenerator: + + def __init__(self): + self.version = coredata.version.replace('dev', '') + self.guid = 'DF5B3ECA-4A31-43E3-8CE4-97FC8A97212E' + self.update_guid = '141527EE-E28A-4D14-97A4-92E6075D28B2' + self.main_xml = 'Meson.wxs' + self.main_o = 'Meson.wixobj' + self.bytesize = '32' if '32' in platform.architecture()[0] else '64' + self.final_output = 'meson-%s-%s.msi' % (self.version, self.bytesize) + self.staging_dir = 'dist' + + def build_dist(self): + if os.path.exists(self.staging_dir): + shutil.rmtree(self.staging_dir) + modules = [os.path.splitext(os.path.split(x)[1])[0] for x in glob(os.path.join('mesonbuild/modules/*'))] + modules = ['mesonbuild.modules.' + x for x in modules if not x.startswith('_')] + modulestr = ','.join(modules) + subprocess.check_call(['c:\\Python\python.exe', + 'c:\\Python\Scripts\\cxfreeze', + '--target-dir', + self.staging_dir, + '--include-modules', + modulestr, + 'meson.py']) + shutil.copy(shutil.which('ninja'), self.staging_dir) + if not os.path.exists(os.path.join(self.staging_dir, 'meson.exe')): + sys.exit('Meson exe missing from staging dir.') + if not os.path.exists(os.path.join(self.staging_dir, 'ninja.exe')): + sys.exit('Ninja exe missing from staging dir.') + + + def generate_files(self): + assert(os.path.isdir(self.staging_dir)) + comp_ref_xml = '' + nodes = {} + with open(self.main_xml, 'w') as ofile: + for root, dirs, files in os.walk(self.staging_dir): + cur_node = Node(dirs, files) + nodes[root] = cur_node + ofile.write(xml_templ % (self.guid, self.update_guid, self.version, self.version)) + self.component_num = 0 + self.create_xml(nodes, ofile, self.staging_dir) + for i in range(self.component_num): + comp_ref_xml += comp_ref_templ % ('ApplicationFiles%d' % i) + ofile.write(xml_footer_templ % comp_ref_xml) + + def create_xml(self, nodes, ofile, root): + cur_node = nodes[root] + if cur_node.files: + ofile.write("\n" % (self.component_num, gen_guid())) + if self.component_num == 0: + ofile.write(path_addition_xml) + self.component_num += 1 + for f in cur_node.files: + file_source = os.path.join(root, f).replace('\\', '\\\\') + file_id = os.path.join(root, f).replace('\\', '_').replace('#', '_').replace('-', '_') + ofile.write(file_templ % (file_id, f, file_source)) + ofile.write('\n') + + for dirname in cur_node.dirs: + dir_id = os.path.join(root, dirname).replace('\\', '_').replace('/', '_') + ofile.write('''\n''' % (dir_id, dirname)) + self.create_xml(nodes, ofile, os.path.join(root, dirname)) + ofile.write('\n') + + + def build_package(self): + subprocess.check_call(['c:\\Program Files\\Wix Toolset v3.11\\bin\candle', self.main_xml]) + subprocess.check_call(['c:\\Program Files\\Wix Toolset v3.11\\bin\light', + '-ext', 'WixUIExtension', + '-cultures:en-us', + '-dWixUILicenseRtf=msi\\License.rtf', + '-out', self.final_output, + self.main_o]) + +if __name__ == '__main__': + if not os.path.exists('meson.py'): + sys.exit(print('Run me in the top level source dir.')) + + p = PackageGenerator() + p.build_dist() + p.generate_files() + p.build_package() diff --git a/run_project_tests.py b/run_project_tests.py index d7ad25778..58a0fd5f3 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -21,7 +21,7 @@ from io import StringIO from ast import literal_eval from enum import Enum import tempfile -import mesontest +from mesonbuild import mtest from mesonbuild import environment from mesonbuild import mesonlib from mesonbuild import mlog @@ -287,12 +287,12 @@ def run_test_inprocess(testdir): os.chdir(testdir) test_log_fname = 'meson-logs/testlog.txt' try: - returncode_test = mesontest.run(['--no-rebuild']) + returncode_test = mtest.run(['--no-rebuild']) if os.path.exists(test_log_fname): test_log = open(test_log_fname, errors='ignore').read() else: test_log = '' - returncode_benchmark = mesontest.run(['--no-rebuild', '--benchmark', '--logbase', 'benchmarklog']) + returncode_benchmark = mtest.run(['--no-rebuild', '--benchmark', '--logbase', 'benchmarklog']) finally: sys.stdout = old_stdout sys.stderr = old_stderr diff --git a/run_tests.py b/run_tests.py index 5a52a44d1..b8c506215 100755 --- a/run_tests.py +++ b/run_tests.py @@ -127,7 +127,7 @@ def run_configure_inprocess(commandlist): old_stderr = sys.stderr sys.stderr = mystderr = StringIO() try: - returncode = mesonmain.run(commandlist[0], commandlist[1:]) + returncode = mesonmain.run(commandlist[1:], commandlist[0]) finally: sys.stdout = old_stdout sys.stderr = old_stderr diff --git a/run_unittests.py b/run_unittests.py index 12b5278e9..5782c1d9d 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -416,8 +416,8 @@ class BasePlatformTests(unittest.TestCase): self.backend = getattr(Backend, os.environ.get('MESON_UNIT_TEST_BACKEND', 'ninja')) self.meson_args = [os.path.join(src_root, 'meson.py'), '--backend=' + self.backend.name] self.meson_command = [sys.executable] + self.meson_args - self.mconf_command = [sys.executable, os.path.join(src_root, 'mesonconf.py')] - self.mintro_command = [sys.executable, os.path.join(src_root, 'mesonintrospect.py')] + self.mconf_command = [sys.executable, os.path.join(src_root, 'meson.py'), 'configure'] + self.mintro_command = [sys.executable, os.path.join(src_root, 'meson.py'), 'introspect'] self.mtest_command = [sys.executable, os.path.join(src_root, 'mesontest.py'), '-C', self.builddir] # Backend-specific build commands self.build_command, self.clean_command, self.test_command, self.install_command, \