diff --git a/.appveyor.yml b/.appveyor.yml index 46edbd4d9..2ef722f83 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -8,6 +8,14 @@ environment: compiler: msys2-mingw backend: ninja + - arch: x64 + compiler: msys2-mingw + backend: ninja + + - arch: x64 + compiler: cygwin + backend: ninja + - arch: x86 compiler: msvc2010 backend: ninja @@ -26,14 +34,6 @@ environment: backend: vs2015 BOOST_ROOT: C:\Libraries\Boost_1_59_0 - - arch: x64 - compiler: cygwin - backend: ninja - - - arch: x64 - compiler: msys2-mingw - backend: ninja - - arch: x64 compiler: msvc2017 backend: ninja @@ -78,27 +78,27 @@ install: - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% ) - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% ) + - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python ) - ps: | If($Env:compiler -eq 'msys2-mingw') { If($Env:arch -eq 'x86') { $env:Path = 'C:\msys64\mingw32\bin;' + $env:Path + $env:MESON_PYTHON_PATH = 'C:\msys64\mingw32\bin' + $env:PYTHON = 'python3' + C:\msys64\usr\bin\pacman -S --noconfirm mingw32/mingw-w64-i686-python3 } Else { $env:Path = 'C:\msys64\mingw64\bin;' + $env:Path + $env:MESON_PYTHON_PATH = 'C:\msys64\mingw64\bin' + $env:PYTHON = 'python3' + C:\msys64\usr\bin\pacman -S --noconfirm mingw64/mingw-w64-x86_64-python3 } } - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" ) - - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python ) - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat ) - cmd: if %compiler%==cygwin ( %WRAPPER% which %PYTHON% ) else ( where %PYTHON% ) - # Install additional packages needed for specific builds. - # This was originally downloaded from http://bootstrap.pypa.io/get-pip.py, - # but they have a broken certificate (on Windows) and an HSTS policy that - # forces usage of HTTPS, so we mirror our own copy. - - ps: If($Env:compiler -eq 'msys2-mingw') {(new-object Net.WebClient).DownloadFile('https://nirbheek.in/files/meson/get-pip.py', 'C:\projects\meson\get-pip.py')} # pkg-config is needed for the pkg-config tests on msvc - ps: If($Env:compiler.StartsWith('msvc')) {(new-object net.webclient).DownloadFile('http://nirbheek.in/files/binaries/pkg-config/win32/pkg-config.exe', 'C:\projects\meson\pkg-config.exe')} - - cmd: if %compiler%==msys2-mingw ( %PYTHON% "C:\projects\meson\get-pip.py" ) - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat ) - ps: | If($Env:compiler -like 'msvc*') { diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 067b71936..f7b56bdef 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -238,8 +238,14 @@ class Backend: return obj_list def serialize_executable(self, exe, cmd_args, workdir, env={}, - capture=None): + extra_paths=None, capture=None): import hashlib + if extra_paths is None: + # The callee didn't check if we needed extra paths, so check it here + if mesonlib.is_windows() or mesonlib.is_cygwin(): + extra_paths = self.determine_windows_extra_paths(exe, []) + else: + extra_paths = [] # Can't just use exe.name here; it will likely be run more than once if isinstance(exe, (dependencies.ExternalProgram, build.BuildTarget, build.CustomTarget)): @@ -272,10 +278,6 @@ class Backend: exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None) else: exe_wrapper = None - if mesonlib.is_windows() or mesonlib.is_cygwin(): - extra_paths = self.determine_windows_extra_paths(exe) - else: - extra_paths = [] es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env, is_cross_built, exe_wrapper, workdir, extra_paths, capture) @@ -531,23 +533,27 @@ class Backend: args.append(d_arg) return args - def determine_windows_extra_paths(self, target): + def determine_windows_extra_paths(self, target, extra_bdeps): '''On Windows there is no such thing as an rpath. We must determine all locations of DLLs that this exe links to and return them so they can be used in unit tests.''' - if not isinstance(target, build.Executable): - return [] - prospectives = target.get_transitive_link_deps() result = [] + prospectives = [] + if isinstance(target, build.Executable): + prospectives = target.get_transitive_link_deps() + # External deps + for deppath in self.rpaths_for_bundled_shared_libraries(target): + result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath))) + for bdep in extra_bdeps: + prospectives += bdep.get_transitive_link_deps() + # Internal deps for ld in prospectives: if ld == '' or ld == '.': continue dirseg = os.path.join(self.environment.get_build_dir(), self.get_target_dir(ld)) if dirseg not in result: result.append(dirseg) - for deppath in self.rpaths_for_bundled_shared_libraries(target): - result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath))) return result def write_benchmark_file(self, datafile): @@ -578,7 +584,7 @@ class Backend: else: exe_wrapper = None if mesonlib.is_windows() or mesonlib.is_cygwin(): - extra_paths = self.determine_windows_extra_paths(exe) + extra_paths = self.determine_windows_extra_paths(exe, []) else: extra_paths = [] cmd_args = [] diff --git a/mesonbuild/backend/ninjabackend.py b/mesonbuild/backend/ninjabackend.py index bcda603ed..e35a391dc 100644 --- a/mesonbuild/backend/ninjabackend.py +++ b/mesonbuild/backend/ninjabackend.py @@ -504,22 +504,30 @@ int dummy; # Add a dependency on all the outputs of this target for output in d.get_outputs(): elem.add_dep(os.path.join(self.get_target_dir(d), output)) + serialize = False + extra_paths = [] # If the target requires capturing stdout, then use the serialized # executable wrapper to capture that output and save it to a file. - # + if target.capture: + serialize = True # If the command line requires a newline, also use the wrapper, as # ninja does not support them in its build rule syntax. - # + if any('\n' in c for c in cmd): + serialize = True # Windows doesn't have -rpath, so for EXEs that need DLLs built within # the project, we need to set PATH so the DLLs are found. We use # a serialized executable wrapper for that and check if the # CustomTarget command needs extra paths first. - if (target.capture or any('\n' in c for c in cmd) or - ((mesonlib.is_windows() or mesonlib.is_cygwin()) and - self.determine_windows_extra_paths(target.command[0]))): + if mesonlib.is_windows() or mesonlib.is_cygwin(): + extra_bdeps = target.get_transitive_build_target_deps() + extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps) + if extra_paths: + serialize = True + if serialize: exe_data = self.serialize_executable(target.command[0], cmd[1:], # All targets are built from the build dir self.environment.get_build_dir(), + extra_paths=extra_paths, capture=ofilenames[0] if target.capture else None) cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] cmd_type = 'meson_exe.py custom' diff --git a/mesonbuild/backend/vs2010backend.py b/mesonbuild/backend/vs2010backend.py index ea02580c0..e872a0474 100644 --- a/mesonbuild/backend/vs2010backend.py +++ b/mesonbuild/backend/vs2010backend.py @@ -429,9 +429,12 @@ class Vs2010Backend(backends.Backend): # Always use a wrapper because MSBuild eats random characters when # there are many arguments. tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target)) + extra_bdeps = target.get_transitive_build_target_deps() + extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps) exe_data = self.serialize_executable(target.command[0], cmd[1:], # All targets run from the target dir tdir_abs, + extra_paths=extra_paths, capture=ofilenames[0] if target.capture else None) wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data] ET.SubElement(customstep, 'Command').text = ' '.join(self.quote_arguments(wrapper_cmd)) diff --git a/mesonbuild/build.py b/mesonbuild/build.py index 8eb95dc13..7757300c5 100644 --- a/mesonbuild/build.py +++ b/mesonbuild/build.py @@ -1549,6 +1549,24 @@ class CustomTarget(Target): deps.append(c) return deps + def get_transitive_build_target_deps(self): + ''' + Recursively fetch the build targets that this custom target depends on, + whether through `command:`, `depends:`, or `sources:` The recursion is + only performed on custom targets. + This is useful for setting PATH on Windows for finding required DLLs. + F.ex, if you have a python script that loads a C module that links to + other DLLs in your project. + ''' + bdeps = set() + deps = self.get_target_dependencies() + for d in deps: + if isinstance(d, BuildTarget): + bdeps.add(d) + elif isinstance(d, CustomTarget): + bdeps.update(d.get_transitive_build_target_deps()) + return bdeps + def flatten_command(self, cmd): cmd = listify(cmd, unholder=True) final_cmd = [] diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py index 61dd953de..ce9313e80 100644 --- a/mesonbuild/dependencies/misc.py +++ b/mesonbuild/dependencies/misc.py @@ -21,6 +21,8 @@ import shlex import shutil import sysconfig +from pathlib import Path + from .. import mlog from .. import mesonlib from ..mesonlib import Popen_safe, extract_as_list @@ -603,6 +605,7 @@ class Python3Dependency(ExternalDependency): def __init__(self, environment, kwargs): super().__init__('python3', environment, None, kwargs) self.name = 'python3' + self.static = kwargs.get('static', False) # We can only be sure that it is Python 3 at this point self.version = '3' self.pkgdep = None @@ -638,12 +641,55 @@ class Python3Dependency(ExternalDependency): else: mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.red('NO')) + @staticmethod + def get_windows_python_arch(): + pyplat = sysconfig.get_platform() + if pyplat == 'mingw': + pycc = sysconfig.get_config_var('CC') + if pycc.startswith('x86_64'): + return '64' + elif pycc.startswith(('i686', 'i386')): + return '32' + else: + mlog.log('MinGW Python built with unknown CC {!r}, please file' + 'a bug'.format(pycc)) + return None + elif pyplat == 'win32': + return '32' + elif pyplat in ('win64', 'win-amd64'): + return '64' + mlog.log('Unknown Windows Python platform {!r}'.format(pyplat)) + return None + + def get_windows_link_args(self): + pyplat = sysconfig.get_platform() + if pyplat.startswith('win'): + vernum = sysconfig.get_config_var('py_version_nodot') + if self.static: + libname = 'libpython{}.a'.format(vernum) + else: + libname = 'python{}.lib'.format(vernum) + lib = Path(sysconfig.get_config_var('base')) / 'libs' / libname + elif pyplat == 'mingw': + if self.static: + libname = sysconfig.get_config_var('LIBRARY') + else: + libname = sysconfig.get_config_var('LDLIBRARY') + lib = Path(sysconfig.get_config_var('LIBDIR')) / libname + if not lib.exists(): + mlog.log('Could not find Python3 library {!r}'.format(str(lib))) + return None + return [str(lib)] + def _find_libpy3_windows(self, env): ''' Find python3 libraries on Windows and also verify that the arch matches what we are building for. ''' - pyarch = sysconfig.get_platform() + pyarch = self.get_windows_python_arch() + if pyarch is None: + self.is_found = False + return arch = detect_cpu_family(env.coredata.compilers) if arch == 'x86': arch = '32' @@ -656,22 +702,24 @@ class Python3Dependency(ExternalDependency): self.is_found = False return # Pyarch ends in '32' or '64' - if arch != pyarch[-2:]: - mlog.log('Need', mlog.bold(self.name), - 'for {}-bit, but found {}-bit'.format(arch, pyarch[-2:])) + if arch != pyarch: + mlog.log('Need', mlog.bold(self.name), 'for {}-bit, but ' + 'found {}-bit'.format(arch, pyarch)) + self.is_found = False + return + # This can fail if the library is not found + largs = self.get_windows_link_args() + if largs is None: self.is_found = False return + self.link_args = largs + # Compile args inc = sysconfig.get_path('include') platinc = sysconfig.get_path('platinclude') self.compile_args = ['-I' + inc] if inc != platinc: self.compile_args.append('-I' + platinc) - # Nothing exposes this directly that I coulf find - basedir = sysconfig.get_config_var('base') - vernum = sysconfig.get_config_var('py_version_nodot') - self.link_args = ['-L{}/libs'.format(basedir), - '-lpython{}'.format(vernum)] - self.version = sysconfig.get_config_var('py_version_short') + self.version = sysconfig.get_config_var('py_version') self.is_found = True def get_methods(self): diff --git a/test cases/python3/4 custom target depends extmodule/blaster.py b/test cases/python3/4 custom target depends extmodule/blaster.py new file mode 100644 index 000000000..49be46644 --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/blaster.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse + +from pathlib import Path + +filedir = Path(os.path.dirname(__file__)).resolve() +if list(filedir.glob('ext/*tachyon.*')): + sys.path.insert(0, (filedir / 'ext').as_posix()) + +import tachyon + +parser = argparse.ArgumentParser() +parser.add_argument('-o', dest='output', default=None) + +options = parser.parse_args(sys.argv[1:]) + +result = tachyon.phaserize('shoot') + +if options.output: + with open(options.output, 'w') as f: + f.write('success') + +if not isinstance(result, int): + print('Returned result not an integer.') + sys.exit(1) + +if result != 1: + print('Returned result {} is not 1.'.format(result)) + sys.exit(1) diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c new file mode 100644 index 000000000..aeff29655 --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c @@ -0,0 +1,8 @@ +#ifdef _MSC_VER +__declspec(dllexport) +#endif +const char* +tachyon_phaser_command (void) +{ + return "shoot"; +} diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h new file mode 100644 index 000000000..dca71d311 --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h @@ -0,0 +1,6 @@ +#pragma once + +#ifdef _MSC_VER +__declspec(dllimport) +#endif +const char* tachyon_phaser_command (void); diff --git a/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build b/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build new file mode 100644 index 000000000..b1f893849 --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/ext/lib/meson.build @@ -0,0 +1,4 @@ +libtachyon = shared_library('tachyonlib', 'meson-tachyonlib.c') + +libtachyon_dep = declare_dependency(link_with : libtachyon, + include_directories : include_directories('.')) diff --git a/test cases/python3/4 custom target depends extmodule/ext/meson.build b/test cases/python3/4 custom target depends extmodule/ext/meson.build new file mode 100644 index 000000000..5ccbe223c --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/ext/meson.build @@ -0,0 +1,6 @@ +subdir('lib') + +pylib = py3_mod.extension_module('tachyon', + 'tachyon_module.c', + dependencies : [libtachyon_dep, py3_dep], +) diff --git a/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c b/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c new file mode 100644 index 000000000..b48032bbb --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c @@ -0,0 +1,51 @@ +/* + Copyright 2016 The Meson development team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* A very simple Python extension module. */ + +#include +#include + +#include "meson-tachyonlib.h" + +static PyObject* phaserize(PyObject *self, PyObject *args) { + const char *message; + int result; + + if(!PyArg_ParseTuple(args, "s", &message)) + return NULL; + + result = strcmp(message, tachyon_phaser_command()) ? 0 : 1; + return PyLong_FromLong(result); +} + +static PyMethodDef TachyonMethods[] = { + {"phaserize", phaserize, METH_VARARGS, + "Shoot tachyon cannons."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef tachyonmodule = { + PyModuleDef_HEAD_INIT, + "tachyon", + NULL, + -1, + TachyonMethods +}; + +PyMODINIT_FUNC PyInit_tachyon(void) { + return PyModule_Create(&tachyonmodule); +} diff --git a/test cases/python3/4 custom target depends extmodule/meson.build b/test cases/python3/4 custom target depends extmodule/meson.build new file mode 100644 index 000000000..dd6fed09a --- /dev/null +++ b/test cases/python3/4 custom target depends extmodule/meson.build @@ -0,0 +1,35 @@ +project('Python extension module', 'c', + default_options : ['buildtype=release']) +# Because Windows Python ships only with optimized libs, +# we must build this project the same way. + +py3_mod = import('python3') +py3 = py3_mod.find_python() +py3_dep = dependency('python3', required : false) + +# Copy to the builddir so that blaster.py can find the built tachyon module +# FIXME: We should automatically detect this case and append the correct paths +# to PYTHONLIBDIR +blaster_py = configure_file(input : 'blaster.py', + output : 'blaster.py', + configuration : configuration_data()) + +check_exists = ''' +import os, sys +with open(sys.argv[1], 'rb') as f: + assert(f.read() == b'success') +''' +if py3_dep.found() + subdir('ext') + + out_txt = custom_target('tachyon flux', + input : blaster_py, + output : 'out.txt', + command : [py3, '@INPUT@', '-o', '@OUTPUT@'], + depends : pylib, + build_by_default: true) + + test('flux', py3, args : ['-c', check_exists, out_txt]) +else + error('MESON_SKIP_TEST: Python3 libraries not found, skipping test.') +endif