diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 982b0ee26..8c2752a77 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -69,17 +69,13 @@ class TargetInstallData: self.optional = optional class ExecutableSerialisation: - def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper, - workdir, extra_paths, capture, needs_exe_wrapper: bool): - self.name = name - self.fname = fname + def __init__(self, cmd_args, env=None, exe_wrapper=None, + workdir=None, extra_paths=None, capture=None): self.cmd_args = cmd_args - self.env = env - self.is_cross = is_cross + self.env = env or {} if exe_wrapper is not None: assert(isinstance(exe_wrapper, dependencies.ExternalProgram)) self.exe_runner = exe_wrapper - self.needs_exe_wrapper = needs_exe_wrapper self.workdir = workdir self.extra_paths = extra_paths self.capture = capture @@ -323,26 +319,61 @@ class Backend: raise MesonException('Unknown data type in object list.') return obj_list - def serialize_executable(self, tname, exe, cmd_args, workdir, env=None, - extra_paths=None, capture=None): + def as_meson_exe_cmdline(self, tname, exe, cmd_args, workdir=None, + for_machine=MachineChoice.BUILD, + extra_bdeps=None, capture=None, force_serialize=False): ''' Serialize an executable for running with a generator or a custom target ''' import hashlib - if env is None: - env = {} - if extra_paths is None: - # The callee didn't check if we needed extra paths, so check it here - if mesonlib.is_windows() or mesonlib.is_cygwin(): - extra_paths = self.determine_windows_extra_paths(exe, []) - else: - extra_paths = [] - # Can't just use exe.name here; it will likely be run more than once + machine = self.environment.machines[for_machine] + if machine.is_windows() or machine.is_cygwin(): + extra_paths = self.determine_windows_extra_paths(exe, extra_bdeps or []) + else: + extra_paths = [] + + if isinstance(exe, dependencies.ExternalProgram): + exe_cmd = exe.get_command() + exe_for_machine = exe.for_machine + elif isinstance(exe, (build.BuildTarget, build.CustomTarget)): + exe_cmd = [self.get_target_filename_abs(exe)] + exe_for_machine = exe.for_machine + else: + exe_cmd = [exe] + exe_for_machine = MachineChoice.BUILD + + is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine) + if is_cross_built and self.environment.need_exe_wrapper(): + exe_wrapper = self.environment.get_exe_wrapper() + if not exe_wrapper.found(): + msg = 'The exe_wrapper {!r} defined in the cross file is ' \ + 'needed by target {!r}, but was not found. Please ' \ + 'check the command and/or add it to PATH.' + raise MesonException(msg.format(exe_wrapper.name, tname)) + else: + if exe_cmd[0].endswith('.jar'): + exe_cmd = ['java', '-jar'] + exe_cmd + elif exe_cmd[0].endswith('.exe') and not (mesonlib.is_windows() or mesonlib.is_cygwin()): + exe_cmd = ['mono'] + exe_cmd + exe_wrapper = None + + force_serialize = force_serialize or extra_paths or workdir or \ + exe_wrapper or any('\n' in c for c in cmd_args) + if not force_serialize: + if not capture: + return None + return (self.environment.get_build_command() + + ['--internal', 'exe', '--capture', capture, '--'] + exe_cmd + cmd_args) + + workdir = workdir or self.environment.get_build_dir() + env = {} if isinstance(exe, (dependencies.ExternalProgram, build.BuildTarget, build.CustomTarget)): basename = exe.name else: basename = os.path.basename(exe) + + # Can't just use exe.name here; it will likely be run more than once # Take a digest of the cmd args, env, workdir, and capture. This avoids # collisions and also makes the name deterministic over regenerations # which avoids a rebuild by Ninja because the cmdline stays the same. @@ -352,31 +383,11 @@ class Backend: scratch_file = 'meson_exe_{0}_{1}.dat'.format(basename, digest) exe_data = os.path.join(self.environment.get_scratch_dir(), scratch_file) with open(exe_data, 'wb') as f: - if isinstance(exe, dependencies.ExternalProgram): - exe_cmd = exe.get_command() - exe_for_machine = exe.for_machine - elif isinstance(exe, (build.BuildTarget, build.CustomTarget)): - exe_cmd = [self.get_target_filename_abs(exe)] - exe_for_machine = exe.for_machine - else: - exe_cmd = [exe] - exe_for_machine = MachineChoice.BUILD - is_cross_built = not self.environment.machines.matches_build_machine(exe_for_machine) - if is_cross_built and self.environment.need_exe_wrapper(): - exe_wrapper = self.environment.get_exe_wrapper() - if not exe_wrapper.found(): - msg = 'The exe_wrapper {!r} defined in the cross file is ' \ - 'needed by target {!r}, but was not found. Please ' \ - 'check the command and/or add it to PATH.' - raise MesonException(msg.format(exe_wrapper.name, tname)) - else: - exe_wrapper = None - es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env, - is_cross_built, exe_wrapper, workdir, - extra_paths, capture, - self.environment.need_exe_wrapper()) + es = ExecutableSerialisation(exe_cmd + cmd_args, env, + exe_wrapper, workdir, + extra_paths, capture) pickle.dump(es, f) - return exe_data + return self.environment.get_build_command() + ['--internal', 'exe', '--unpickle', exe_data] def serialize_tests(self): test_data = os.path.join(self.environment.get_scratch_dir(), 'meson_test_setup.dat') diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index 570a153c5..b614b4133 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -658,33 +658,13 @@ int dummy; # Add a dependency on all the outputs of this target for output in d.get_outputs(): elem.add_dep(os.path.join(self.get_target_dir(d), output)) - serialize = False - extra_paths = [] - # If the target requires capturing stdout, then use the serialized - # executable wrapper to capture that output and save it to a file. - if target.capture: - serialize = True - # If the command line requires a newline, also use the wrapper, as - # ninja does not support them in its build rule syntax. - if any('\n' in c for c in cmd): - serialize = True - # Windows doesn't have -rpath, so for EXEs that need DLLs built within - # the project, we need to set PATH so the DLLs are found. We use - # a serialized executable wrapper for that and check if the - # CustomTarget command needs extra paths first. - machine = self.environment.machines[target.for_machine] - if machine.is_windows() or machine.is_cygwin(): - extra_bdeps = target.get_transitive_build_target_deps() - extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps) - if extra_paths: - serialize = True - if serialize: - exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:], - # All targets are built from the build dir - self.environment.get_build_dir(), - extra_paths=extra_paths, - capture=ofilenames[0] if target.capture else None) - cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] + + meson_exe_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:], + for_machine=target.for_machine, + extra_bdeps=target.get_transitive_build_target_deps(), + capture=ofilenames[0] if target.capture else None) + if meson_exe_cmd: + cmd = meson_exe_cmd cmd_type = 'meson_exe.py custom' else: cmd_type = 'custom' @@ -1786,19 +1766,13 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) outfilelist = outfilelist[len(generator.outputs):] args = self.replace_paths(target, args, override_subdir=subdir) cmdlist = exe_arr + self.replace_extra_args(args, genlist) - if generator.capture: - exe_data = self.serialize_executable( - 'generator ' + cmdlist[0], - cmdlist[0], - cmdlist[1:], - self.environment.get_build_dir(), - capture=outfiles[0] - ) - cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] - abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) - os.makedirs(abs_pdir, exist_ok=True) - else: - cmd = cmdlist + meson_exe_cmd = self.as_meson_exe_cmdline('generator ' + cmdlist[0], + cmdlist[0], cmdlist[1:], + capture=outfiles[0] if generator.capture else None) + if meson_exe_cmd: + cmdlist = meson_exe_cmd + abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) + os.makedirs(abs_pdir, exist_ok=True) elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename) elem.add_dep([self.get_target_filename(x) for x in generator.depends]) @@ -1813,7 +1787,7 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485''')) elem.add_item('DESC', 'Generating source from {!r}.'.format(sole_output)) if isinstance(exe, build.BuildTarget): elem.add_dep(self.get_target_filename(exe)) - elem.add_item('COMMAND', cmd) + elem.add_item('COMMAND', cmdlist) self.add_build(elem) def scan_fortran_module_outputs(self, target): diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index 82fc0cf40..a0f6a95b0 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -143,22 +143,24 @@ class Vs2010Backend(backends.Backend): for x in args] args = [x.replace('\\', '/') for x in args] cmd = exe_arr + self.replace_extra_args(args, genlist) - if generator.capture: - exe_data = self.serialize_executable( - 'generator ' + cmd[0], - cmd[0], - cmd[1:], - self.environment.get_build_dir(), - capture=outfiles[0] - ) - cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] - abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) - os.makedirs(abs_pdir, exist_ok=True) + # Always use a wrapper because MSBuild eats random characters when + # there are many arguments. + tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) + cmd = self.as_meson_exe_cmdline( + 'generator ' + cmd[0], + cmd[0], + cmd[1:], + workdir=tdir_abs, + capture=outfiles[0] if generator.capture else None, + force_serialize=True + ) + deps = cmd[-1:] + deps + abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) + os.makedirs(abs_pdir, exist_ok=True) cbs = ET.SubElement(idgroup, 'CustomBuild', Include=infilename) ET.SubElement(cbs, 'Command').text = ' '.join(self.quote_arguments(cmd)) ET.SubElement(cbs, 'Outputs').text = ';'.join(outfiles) - if deps: - ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps) + ET.SubElement(cbs, 'AdditionalInputs').text = ';'.join(deps) return generator_output_files, custom_target_output_files, custom_target_include_dirs def generate(self, interp): @@ -558,19 +560,18 @@ class Vs2010Backend(backends.Backend): # there are many arguments. tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) extra_bdeps = target.get_transitive_build_target_deps() - extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps) - exe_data = self.serialize_executable(target.name, target.command[0], cmd[1:], - # All targets run from the target dir - tdir_abs, - extra_paths=extra_paths, - capture=ofilenames[0] if target.capture else None) - wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] + wrapper_cmd = self.as_meson_exe_cmdline(target.name, target.command[0], cmd[1:], + # All targets run from the target dir + workdir=tdir_abs, + extra_bdeps=extra_bdeps, + capture=ofilenames[0] if target.capture else None, + force_serialize=True) if target.build_always_stale: # Use a nonexistent file to always consider the target out-of-date. ofilenames += [self.nonexistent_file(os.path.join(self.environment.get_scratch_dir(), 'outofdate.file'))] self.add_custom_build(root, 'custom_target', ' '.join(self.quote_arguments(wrapper_cmd)), - deps=[exe_data] + srcs + depend_files, outputs=ofilenames) + deps=wrapper_cmd[-1:] + srcs + depend_files, outputs=ofilenames) ET.SubElement(root, 'Import', Project=r'$(VCTargetsPath)\Microsoft.Cpp.targets') self.generate_custom_generator_commands(target, root) self.add_regen_dependency(root) diff --git a/mesonbuild/scripts/meson_exe.py b/mesonbuild/scripts/meson_exe.py index c1d0d64e9..8b34448a5 100644 --- a/mesonbuild/scripts/meson_exe.py +++ b/mesonbuild/scripts/meson_exe.py @@ -20,12 +20,14 @@ import platform import subprocess from .. import mesonlib +from ..backend.backends import ExecutableSerialisation options = None def buildparser(): - parser = argparse.ArgumentParser() - parser.add_argument('args', nargs='+') + parser = argparse.ArgumentParser(description='Custom executable wrapper for Meson. Do not run on your own, mmm\'kay?') + parser.add_argument('--unpickle') + parser.add_argument('--capture') return parser def is_windows(): @@ -36,28 +38,14 @@ def is_cygwin(): platname = platform.system().lower() return 'cygwin' in platname -def run_with_mono(fname): - if fname.endswith('.exe') and not (is_windows() or is_cygwin()): - return True - return False - def run_exe(exe): - if exe.fname[0].endswith('.jar'): - cmd = ['java', '-jar'] + exe.fname - elif not exe.is_cross and run_with_mono(exe.fname[0]): - cmd = ['mono'] + exe.fname + if exe.exe_runner: + if not exe.exe_runner.found(): + raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found ' + 'wrapper {!r}'.format(exe.cmd_args[0], exe.exe_runner.get_path())) + cmd_args = exe.exe_runner.get_command() + exe.cmd_args else: - if exe.is_cross and exe.needs_exe_wrapper: - if exe.exe_runner is None: - raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} ' - 'with no wrapper'.format(exe.name)) - elif not exe.exe_runner.found(): - raise AssertionError('BUG: Can\'t run cross-compiled exe {!r} with not-found ' - 'wrapper {!r}'.format(exe.name, exe.exe_runner.get_path())) - else: - cmd = exe.exe_runner.get_command() + exe.fname - else: - cmd = exe.fname + cmd_args = exe.cmd_args child_env = os.environ.copy() child_env.update(exe.env) if exe.extra_paths: @@ -73,7 +61,7 @@ def run_exe(exe): else: child_env['WINEPATH'] = wine_path - p = subprocess.Popen(cmd + exe.cmd_args, env=child_env, cwd=exe.workdir, + p = subprocess.Popen(cmd_args, env=child_env, cwd=exe.workdir, close_fds=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -101,13 +89,22 @@ def run_exe(exe): def run(args): global options - options = buildparser().parse_args(args) - if len(options.args) != 1: - print('Test runner for Meson. Do not run on your own, mmm\'kay?') - print(sys.argv[0] + ' [data file]') - exe_data_file = options.args[0] - with open(exe_data_file, 'rb') as f: - exe = pickle.load(f) + parser = buildparser() + options, cmd_args = parser.parse_known_args(args) + # argparse supports double dash to separate options and positional arguments, + # but the user has to remove it manually. + if cmd_args and cmd_args[0] == '--': + cmd_args = cmd_args[1:] + if not options.unpickle and not cmd_args: + parser.error('either --unpickle or executable and arguments are required') + if options.unpickle: + if cmd_args or options.capture: + parser.error('no other arguments can be used with --unpickle') + with open(options.unpickle, 'rb') as f: + exe = pickle.load(f) + else: + exe = ExecutableSerialisation(cmd_args, capture=options.capture) + return run_exe(exe) if __name__ == '__main__': diff --git a/test cases/common/185 escape and unicode/file.py b/test cases/common/185 escape and unicode/file.py index af67a0950..40fa7ca48 100644 --- a/test cases/common/185 escape and unicode/file.py +++ b/test cases/common/185 escape and unicode/file.py @@ -6,5 +6,5 @@ import os with open(sys.argv[1]) as fh: content = fh.read().replace("{NAME}", sys.argv[2]) -with open(os.path.join(sys.argv[3]), 'w') as fh: +with open(os.path.join(sys.argv[3]), 'w', errors='replace') as fh: fh.write(content)