From ec286d497085a297bac12514ef2ddeb730ebd9bc Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Mon, 11 Sep 2017 20:55:37 +0300 Subject: [PATCH 1/6] Change an error message when find_program() fails. Mention all the sought programs in the error message. --- mesonbuild/interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 5c333a885..c937a5ad7 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2169,7 +2169,7 @@ to directly access options of other subprojects.''') if progobj is None: progobj = self.program_from_system(args) if required and (progobj is None or not progobj.found()): - raise InvalidArguments('Program "%s" not found or not executable' % args[0]) + raise InvalidArguments('Program(s) {!r} not found or not executable'.format(args)) if progobj is None: return ExternalProgramHolder(dependencies.NonExistingExternalProgram()) return progobj From abcaf7c222c9e74774f7f5396e16cac40774c236 Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Sun, 10 Sep 2017 10:15:50 +0300 Subject: [PATCH 2/6] Namespace test setups. Use $project_name:$test_setup namespace scheme for test setups. This allows one to choose from which (sub)project a test setup is taken from should there be several sharing the same name. Defaults to the main project. E.g. "meson test --setup subproj:valgrind". --- mesonbuild/interpreter.py | 16 +++++++--------- mesonbuild/mtest.py | 9 +++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index c937a5ad7..0ef1587b5 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -2892,8 +2892,10 @@ different subdirectory. if len(args) != 1: raise InterpreterException('Add_test_setup needs one argument for the setup name.') setup_name = args[0] - if re.fullmatch('[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None: + if re.fullmatch('([_a-zA-Z][_0-9a-zA-Z]*:)?[_a-zA-Z][_0-9a-zA-Z]*', setup_name) is None: raise InterpreterException('Setup name may only contain alphanumeric characters.') + if ":" not in setup_name: + setup_name = (self.subproject if self.subproject else self.build.project_name) + ":" + setup_name try: inp = extract_as_list(kwargs, 'exe_wrapper') exe_wrapper = [] @@ -2917,14 +2919,10 @@ different subdirectory. if not isinstance(timeout_multiplier, int): raise InterpreterException('Timeout multiplier must be a number.') env = self.unpack_env_kwarg(kwargs) - setupobj = build.TestSetup(exe_wrapper=exe_wrapper, - gdb=gdb, - timeout_multiplier=timeout_multiplier, - env=env) - if self.subproject == '': - # Dunno what we should do with subprojects really. Let's start simple - # and just use the master project ones. - self.build.test_setups[setup_name] = setupobj + self.build.test_setups[setup_name] = build.TestSetup(exe_wrapper=exe_wrapper, + gdb=gdb, + timeout_multiplier=timeout_multiplier, + env=env) @permittedKwargs(permitted_kwargs['add_global_arguments']) @stringArgs diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index a6971063b..8480e3d69 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -446,7 +446,7 @@ TIMEOUT: %4d if self.options.wrapper: namebase = os.path.basename(self.get_wrapper()[0]) elif self.options.setup: - namebase = self.options.setup + namebase = self.options.setup.replace(":", "_") if namebase: logfile_base += '-' + namebase.replace(' ', '_') @@ -553,10 +553,11 @@ def merge_suite_options(options): buildfile = os.path.join(options.wd, 'meson-private/build.dat') with open(buildfile, 'rb') as f: build = pickle.load(f) - setups = build.test_setups - if options.setup not in setups: + if ":" not in options.setup: + options.setup = (build.subproject if build.subproject else build.project_name) + ":" + options.setup + if options.setup not in build.test_setups: sys.exit('Unknown test setup: %s' % options.setup) - current = setups[options.setup] + current = build.test_setups[options.setup] if not options.gdb: options.gdb = current.gdb if options.timeout_multiplier is None: From 46110976548cc56153c6186bc16210998758a7c3 Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Mon, 11 Sep 2017 01:04:50 +0300 Subject: [PATCH 3/6] Include project name in a test object. --- mesonbuild/backend/backends.py | 11 ++++++----- mesonbuild/interpreter.py | 11 +++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index b547bc3e7..a8e816484 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -65,9 +65,10 @@ class ExecutableSerialisation: self.capture = capture class TestSerialisation: - def __init__(self, name, suite, fname, is_cross_built, exe_wrapper, is_parallel, cmd_args, env, - should_fail, timeout, workdir, extra_paths): + def __init__(self, name, project, suite, fname, is_cross_built, exe_wrapper, is_parallel, + cmd_args, env, should_fail, timeout, workdir, extra_paths): self.name = name + self.project_name = project self.suite = suite self.fname = fname self.is_cross_built = is_cross_built @@ -603,9 +604,9 @@ class Backend: cmd_args.append(self.get_target_filename(a)) else: raise MesonException('Bad object in test command.') - ts = TestSerialisation(t.get_name(), t.suite, cmd, is_cross, exe_wrapper, - t.is_parallel, cmd_args, t.env, t.should_fail, - t.timeout, t.workdir, extra_paths) + ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross, + exe_wrapper, t.is_parallel, cmd_args, t.env, + t.should_fail, t.timeout, t.workdir, extra_paths) arr.append(ts) pickle.dump(arr, datafile) diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index 0ef1587b5..e3680acd0 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -653,10 +653,11 @@ class RunTargetHolder(InterpreterObject, ObjectHolder): return r.format(self.__class__.__name__, h.get_id(), h.command) class Test(InterpreterObject): - def __init__(self, name, suite, exe, is_parallel, cmd_args, env, should_fail, timeout, workdir): + def __init__(self, name, project, suite, exe, is_parallel, cmd_args, env, should_fail, timeout, workdir): InterpreterObject.__init__(self) self.name = name self.suite = suite + self.project_name = project self.exe = exe self.is_parallel = is_parallel self.cmd_args = cmd_args @@ -2586,14 +2587,12 @@ root and issuing %s. if not isinstance(timeout, int): raise InterpreterException('Timeout must be an integer.') suite = [] + prj = self.subproject if self.is_subproject() else self.build.project_name for s in mesonlib.stringlistify(kwargs.get('suite', '')): if len(s) > 0: s = ':' + s - if self.is_subproject(): - suite.append(self.subproject.replace(' ', '_').replace(':', '_') + s) - else: - suite.append(self.build.project_name.replace(' ', '_').replace(':', '_') + s) - t = Test(args[0], suite, exe.held_object, par, cmd_args, env, should_fail, timeout, workdir) + suite.append(prj.replace(' ', '_').replace(':', '_') + s) + t = Test(args[0], prj, suite, exe.held_object, par, cmd_args, env, should_fail, timeout, workdir) if is_base_test: self.build.tests.append(t) mlog.debug('Adding test "', mlog.bold(args[0]), '".', sep='') From 060560bf6250fcf9b2b528226176d322af93711b Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Mon, 11 Sep 2017 20:36:38 +0300 Subject: [PATCH 4/6] Use test setups from the active (sub)project by default. Replace the logic where a test setup with no project specifier defaults to the main project with one that takes the test setup from the same (sub)project from where the to-be-executed test has been read from. --- mesonbuild/mtest.py | 84 ++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 8480e3d69..686ddfcde 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -28,6 +28,7 @@ import concurrent.futures as conc import platform import signal import random +from copy import deepcopy # GNU autotools interprets a return code of 77 from tests it executes to # mean that the test should be skipped. @@ -89,7 +90,7 @@ parser.add_argument('-v', '--verbose', default=False, action='store_true', help='Do not redirect stdout and stderr') parser.add_argument('-q', '--quiet', default=False, action='store_true', help='Produce less output to the terminal.') -parser.add_argument('-t', '--timeout-multiplier', type=float, default=None, +parser.add_argument('-t', '--timeout-multiplier', type=float, default=1, help='Define a multiplier for test timeout, for example ' ' when running tests in particular conditions they might take' ' more time to execute.') @@ -192,7 +193,17 @@ class TestHarness: if self.jsonlogfile: self.jsonlogfile.close() - def run_single_test(self, wrap, test): + def get_test_env(self, options, test): + if options.setup: + env = merge_suite_options(options, test) + else: + env = os.environ.copy() + if isinstance(test.env, build.EnvironmentVariables): + test.env = test.env.get_env(env) + env.update(test.env) + return env + + def run_single_test(self, test): if test.fname[0].endswith('.jar'): cmd = ['java', '-jar'] + test.fname elif not test.is_cross_built and run_with_mono(test.fname[0]): @@ -215,24 +226,26 @@ class TestHarness: stde = None returncode = GNU_SKIP_RETURNCODE else: + test_opts = deepcopy(self.options) + test_env = self.get_test_env(test_opts, test) + wrap = self.get_wrapper(test_opts) + + if test_opts.gdb: + test.timeout = None + cmd = wrap + cmd + test.cmd_args + self.options.test_args starttime = time.time() - child_env = os.environ.copy() - child_env.update(self.options.global_env.get_env(child_env)) - if isinstance(test.env, build.EnvironmentVariables): - test.env = test.env.get_env(child_env) - child_env.update(test.env) if len(test.extra_paths) > 0: - child_env['PATH'] = os.pathsep.join(test.extra_paths + ['']) + child_env['PATH'] + test_env['PATH'] = os.pathsep.join(test.extra_paths + ['']) + test_env['PATH'] # If MALLOC_PERTURB_ is not set, or if it is set to an empty value, # (i.e., the test or the environment don't explicitly set it), set # it ourselves. We do this unconditionally for regular tests # because it is extremely useful to have. # Setting MALLOC_PERTURB_="0" will completely disable this feature. - if ('MALLOC_PERTURB_' not in child_env or not child_env['MALLOC_PERTURB_']) and not self.options.benchmark: - child_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) + if ('MALLOC_PERTURB_' not in test_env or not test_env['MALLOC_PERTURB_']) and not self.options.benchmark: + test_env['MALLOC_PERTURB_'] = str(random.randint(1, 255)) setsid = None stdout = None @@ -247,7 +260,7 @@ class TestHarness: p = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, - env=child_env, + env=test_env, cwd=test.workdir, preexec_fn=setsid) timed_out = False @@ -255,7 +268,7 @@ class TestHarness: if test.timeout is None: timeout = None else: - timeout = test.timeout * self.options.timeout_multiplier + timeout = test.timeout * test_opts.timeout_multiplier try: (stdo, stde) = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: @@ -444,7 +457,7 @@ TIMEOUT: %4d logfile_base = os.path.join(self.options.wd, 'meson-logs', self.options.logbase) if self.options.wrapper: - namebase = os.path.basename(self.get_wrapper()[0]) + namebase = os.path.basename(self.get_wrapper(self.options)[0]) elif self.options.setup: namebase = self.options.setup.replace(":", "_") @@ -459,16 +472,16 @@ TIMEOUT: %4d self.logfile.write('Log of Meson test suite run on %s\n\n' % datetime.datetime.now().isoformat()) - def get_wrapper(self): + def get_wrapper(self, options): wrap = [] - if self.options.gdb: + if options.gdb: wrap = ['gdb', '--quiet', '--nh'] - if self.options.repeat > 1: + if options.repeat > 1: wrap += ['-ex', 'run', '-ex', 'quit'] # Signal the end of arguments to gdb wrap += ['--args'] - if self.options.wrapper: - wrap += self.options.wrapper + if options.wrapper: + wrap += options.wrapper assert(isinstance(wrap, list)) return wrap @@ -487,7 +500,6 @@ TIMEOUT: %4d futures = [] numlen = len('%d' % len(tests)) self.open_log_files() - wrap = self.get_wrapper() startdir = os.getcwd() if self.options.wd: os.chdir(self.options.wd) @@ -497,18 +509,15 @@ TIMEOUT: %4d for i, test in enumerate(tests): visible_name = self.get_pretty_suite(test) - if self.options.gdb: - test.timeout = None - if not test.is_parallel or self.options.gdb: self.drain_futures(futures) futures = [] - res = self.run_single_test(wrap, test) + res = self.run_single_test(test) self.print_stats(numlen, tests, visible_name, res, i) else: if not executor: executor = conc.ThreadPoolExecutor(max_workers=self.options.num_processes) - f = executor.submit(self.run_single_test, wrap, test) + f = executor.submit(self.run_single_test, test) futures.append((f, numlen, tests, visible_name, i)) if self.options.repeat > 1 and self.fail_count: break @@ -549,15 +558,19 @@ def list_tests(th): for t in tests: print(th.get_pretty_suite(t)) -def merge_suite_options(options): +def merge_suite_options(options, test): buildfile = os.path.join(options.wd, 'meson-private/build.dat') with open(buildfile, 'rb') as f: build = pickle.load(f) - if ":" not in options.setup: - options.setup = (build.subproject if build.subproject else build.project_name) + ":" + options.setup - if options.setup not in build.test_setups: - sys.exit('Unknown test setup: %s' % options.setup) - current = build.test_setups[options.setup] + if ":" in options.setup: + if options.setup not in build.test_setups: + sys.exit("Unknown test setup '%s'." % options.setup) + current = build.test_setups[options.setup] + else: + full_name = test.project_name + ":" + options.setup + if full_name not in build.test_setups: + sys.exit("Test setup '%s' not found from project '%s'." % (options.setup, test.project_name)) + current = build.test_setups[full_name] if not options.gdb: options.gdb = current.gdb if options.timeout_multiplier is None: @@ -568,7 +581,7 @@ def merge_suite_options(options): sys.exit('Conflict: both test setup and command line specify an exe wrapper.') if options.wrapper is None: options.wrapper = current.exe_wrapper - return current.env + return current.env.get_env(os.environ.copy()) def rebuild_all(wd): if not os.path.isfile(os.path.join(wd, 'build.ninja')): @@ -595,15 +608,6 @@ def run(args): if options.benchmark: options.num_processes = 1 - if options.setup is not None: - global_env = merge_suite_options(options) - else: - global_env = build.EnvironmentVariables() - if options.timeout_multiplier is None: - options.timeout_multiplier = 1 - - setattr(options, 'global_env', global_env) - if options.verbose and options.quiet: print('Can not be both quiet and verbose at the same time.') return 1 From 473dc807017829403eb9868034d21de24c261473 Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Mon, 11 Sep 2017 21:24:24 +0300 Subject: [PATCH 5/6] Add some test setup selection tests. --- run_unittests.py | 25 +++++++++++++++++++ test cases/unit/13 testsetup selection/main.c | 3 +++ .../unit/13 testsetup selection/meson.build | 10 ++++++++ .../subprojects/bar/bar.c | 3 +++ .../subprojects/bar/meson.build | 6 +++++ .../subprojects/foo/foo.c | 3 +++ .../subprojects/foo/meson.build | 4 +++ 7 files changed, 54 insertions(+) create mode 100644 test cases/unit/13 testsetup selection/main.c create mode 100644 test cases/unit/13 testsetup selection/meson.build create mode 100644 test cases/unit/13 testsetup selection/subprojects/bar/bar.c create mode 100644 test cases/unit/13 testsetup selection/subprojects/bar/meson.build create mode 100644 test cases/unit/13 testsetup selection/subprojects/foo/foo.c create mode 100644 test cases/unit/13 testsetup selection/subprojects/foo/meson.build diff --git a/run_unittests.py b/run_unittests.py index 41cbf3931..b69aaae2c 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -949,6 +949,31 @@ class AllPlatformTests(BasePlatformTests): # Setup with only a timeout works self._run(self.mtest_command + ['--setup=timeout']) + def test_testsetup_selection(self): + testdir = os.path.join(self.unit_test_dir, '13 testsetup selection') + self.init(testdir) + self.build() + + # Run tests without setup + self.run_tests() + + self.assertRaises(subprocess.CalledProcessError, self._run, self.mtest_command + ['--setup=missingfromfoo']) + self._run(self.mtest_command + ['--setup=missingfromfoo', '--no-suite=foo:']) + + self._run(self.mtest_command + ['--setup=worksforall']) + self._run(self.mtest_command + ['--setup=main:worksforall']) + + self.assertRaises(subprocess.CalledProcessError, self._run, + self.mtest_command + ['--setup=onlyinbar']) + self.assertRaises(subprocess.CalledProcessError, self._run, + self.mtest_command + ['--setup=onlyinbar', '--no-suite=main:']) + self._run(self.mtest_command + ['--setup=onlyinbar', '--no-suite=main:', '--no-suite=foo:']) + self._run(self.mtest_command + ['--setup=bar:onlyinbar']) + self.assertRaises(subprocess.CalledProcessError, self._run, + self.mtest_command + ['--setup=foo:onlyinbar']) + self.assertRaises(subprocess.CalledProcessError, self._run, + self.mtest_command + ['--setup=main:onlyinbar']) + def assertFailedTestCount(self, failure_count, command): try: self._run(command) diff --git a/test cases/unit/13 testsetup selection/main.c b/test cases/unit/13 testsetup selection/main.c new file mode 100644 index 000000000..cb3f7482f --- /dev/null +++ b/test cases/unit/13 testsetup selection/main.c @@ -0,0 +1,3 @@ +int main() { + return 0; +} diff --git a/test cases/unit/13 testsetup selection/meson.build b/test cases/unit/13 testsetup selection/meson.build new file mode 100644 index 000000000..ae996c5be --- /dev/null +++ b/test cases/unit/13 testsetup selection/meson.build @@ -0,0 +1,10 @@ +project('main', 'c') + +main = executable('main', 'main.c') +test('Test main', main) + +add_test_setup('worksforall') +add_test_setup('missingfromfoo') + +subproject('foo') +subproject('bar') diff --git a/test cases/unit/13 testsetup selection/subprojects/bar/bar.c b/test cases/unit/13 testsetup selection/subprojects/bar/bar.c new file mode 100644 index 000000000..cb3f7482f --- /dev/null +++ b/test cases/unit/13 testsetup selection/subprojects/bar/bar.c @@ -0,0 +1,3 @@ +int main() { + return 0; +} diff --git a/test cases/unit/13 testsetup selection/subprojects/bar/meson.build b/test cases/unit/13 testsetup selection/subprojects/bar/meson.build new file mode 100644 index 000000000..1155a88aa --- /dev/null +++ b/test cases/unit/13 testsetup selection/subprojects/bar/meson.build @@ -0,0 +1,6 @@ +project('bar', 'c') +bar = executable('bar', 'bar.c') +test('Test bar', bar) +add_test_setup('onlyinbar') +add_test_setup('worksforall') +add_test_setup('missingfromfoo') diff --git a/test cases/unit/13 testsetup selection/subprojects/foo/foo.c b/test cases/unit/13 testsetup selection/subprojects/foo/foo.c new file mode 100644 index 000000000..cb3f7482f --- /dev/null +++ b/test cases/unit/13 testsetup selection/subprojects/foo/foo.c @@ -0,0 +1,3 @@ +int main() { + return 0; +} diff --git a/test cases/unit/13 testsetup selection/subprojects/foo/meson.build b/test cases/unit/13 testsetup selection/subprojects/foo/meson.build new file mode 100644 index 000000000..2eef840cd --- /dev/null +++ b/test cases/unit/13 testsetup selection/subprojects/foo/meson.build @@ -0,0 +1,4 @@ +project('foo', 'c') +foo = executable('foo', 'foo.c') +test('Test foo', foo) +add_test_setup('worksforall') From 4f8db1fe47056fa650236c9c89ddd4814a271174 Mon Sep 17 00:00:00 2001 From: Hemmo Nieminen Date: Sun, 25 Feb 2018 20:38:41 +0200 Subject: [PATCH 6/6] Add a release note snippet for the improved test setup selection. --- docs/markdown/snippets/test-setups.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/markdown/snippets/test-setups.md diff --git a/docs/markdown/snippets/test-setups.md b/docs/markdown/snippets/test-setups.md new file mode 100644 index 000000000..4598e7369 --- /dev/null +++ b/docs/markdown/snippets/test-setups.md @@ -0,0 +1,16 @@ +## Improve test setup selection + +Test setups are now identified (also) by the project they belong to and it +is possible to select the used test setup from a specific project. E.g. +to use a test setup `some_setup` from project `some_project` for all +executed tests one can use + + meson test --setup some_project:some_setup + +Should one rather want test setups to be used from the same project as +where the current test itself has been defined, one can use just + + meson test --setup some_setup + +In the latter case every (sub)project must have a test setup `some_setup` +defined in it.