Merge pull request #2697 from mesonbuild/custom-target-depends-serialize

custom target: Consider all build depends while serializing

.appveyor.yml

@@ -8,6 +8,14 @@ environment:
       compiler: msys2-mingw
       backend: ninja
 
+    - arch: x64
+      compiler: msys2-mingw
+      backend: ninja
+
+    - arch: x64
+      compiler: cygwin
+      backend: ninja
+
     - arch: x86
       compiler: msvc2010
       backend: ninja
@@ -26,14 +34,6 @@ environment:
       backend: vs2015
       BOOST_ROOT: C:\Libraries\Boost_1_59_0
 
-    - arch: x64
-      compiler: cygwin
-      backend: ninja
-
-    - arch: x64
-      compiler: msys2-mingw
-      backend: ninja
-
     - arch: x64
       compiler: msvc2017
       backend: ninja
@@ -78,27 +78,27 @@ install:
   - cmd: if %compiler%==msvc2010 ( call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2015 ( call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %arch% )
   - cmd: if %compiler%==msvc2017 ( call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=%arch% )
+  - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python )
   - ps: |
         If($Env:compiler -eq 'msys2-mingw') {
           If($Env:arch -eq 'x86') {
             $env:Path = 'C:\msys64\mingw32\bin;' + $env:Path
+            $env:MESON_PYTHON_PATH = 'C:\msys64\mingw32\bin'
+            $env:PYTHON = 'python3'
+            C:\msys64\usr\bin\pacman -S --noconfirm mingw32/mingw-w64-i686-python3
           } Else {
             $env:Path = 'C:\msys64\mingw64\bin;' + $env:Path
+            $env:MESON_PYTHON_PATH = 'C:\msys64\mingw64\bin'
+            $env:PYTHON = 'python3'
+            C:\msys64\usr\bin\pacman -S --noconfirm mingw64/mingw-w64-x86_64-python3
           }
         }
   - cmd: if not %compiler%==cygwin ( set "PATH=%cd%;%MESON_PYTHON_PATH%;%PATH%;" )
-  - cmd: if %compiler%==cygwin ( set PYTHON=python3 ) else ( set PYTHON=python )
   - cmd: if %compiler%==cygwin ( set WRAPPER=ci\run-in-cygwin.bat )
   - cmd: if %compiler%==cygwin ( %WRAPPER% which %PYTHON% ) else ( where %PYTHON% )
 
-  # Install additional packages needed for specific builds.
-  # This was originally downloaded from http://bootstrap.pypa.io/get-pip.py,
-  # but they have a broken certificate (on Windows) and an HSTS policy that
-  # forces usage of HTTPS, so we mirror our own copy.
-  - ps: If($Env:compiler -eq 'msys2-mingw') {(new-object Net.WebClient).DownloadFile('https://nirbheek.in/files/meson/get-pip.py', 'C:\projects\meson\get-pip.py')}
   # pkg-config is needed for the pkg-config tests on msvc
   - ps: If($Env:compiler.StartsWith('msvc')) {(new-object net.webclient).DownloadFile('http://nirbheek.in/files/binaries/pkg-config/win32/pkg-config.exe', 'C:\projects\meson\pkg-config.exe')}
-  - cmd: if %compiler%==msys2-mingw ( %PYTHON% "C:\projects\meson\get-pip.py" )
   - cmd: if %compiler%==cygwin ( call ci\appveyor-install.bat )
   - ps: |
         If($Env:compiler -like 'msvc*') {

mesonbuild/backend/backends.py

@@ -238,8 +238,14 @@ class Backend:
         return obj_list
 
     def serialize_executable(self, exe, cmd_args, workdir, env={},
-                             capture=None):
+                             extra_paths=None, capture=None):
         import hashlib
+        if extra_paths is None:
+            # The callee didn't check if we needed extra paths, so check it here
+            if mesonlib.is_windows() or mesonlib.is_cygwin():
+                extra_paths = self.determine_windows_extra_paths(exe, [])
+            else:
+                extra_paths = []
         # Can't just use exe.name here; it will likely be run more than once
         if isinstance(exe, (dependencies.ExternalProgram,
                             build.BuildTarget, build.CustomTarget)):
@@ -272,10 +278,6 @@ class Backend:
                 exe_wrapper = self.environment.cross_info.config['binaries'].get('exe_wrapper', None)
             else:
                 exe_wrapper = None
-            if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
-            else:
-                extra_paths = []
             es = ExecutableSerialisation(basename, exe_cmd, cmd_args, env,
                                          is_cross_built, exe_wrapper, workdir,
                                          extra_paths, capture)
@@ -531,23 +533,27 @@ class Backend:
             args.append(d_arg)
         return args
 
-    def determine_windows_extra_paths(self, target):
+    def determine_windows_extra_paths(self, target, extra_bdeps):
         '''On Windows there is no such thing as an rpath.
         We must determine all locations of DLLs that this exe
         links to and return them so they can be used in unit
         tests.'''
-        if not isinstance(target, build.Executable):
-            return []
-        prospectives = target.get_transitive_link_deps()
         result = []
+        prospectives = []
+        if isinstance(target, build.Executable):
+            prospectives = target.get_transitive_link_deps()
+            # External deps
+            for deppath in self.rpaths_for_bundled_shared_libraries(target):
+                result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
+        for bdep in extra_bdeps:
+            prospectives += bdep.get_transitive_link_deps()
+        # Internal deps
         for ld in prospectives:
             if ld == '' or ld == '.':
                 continue
             dirseg = os.path.join(self.environment.get_build_dir(), self.get_target_dir(ld))
             if dirseg not in result:
                 result.append(dirseg)
-        for deppath in self.rpaths_for_bundled_shared_libraries(target):
-            result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
         return result
 
     def write_benchmark_file(self, datafile):
@@ -578,7 +584,7 @@ class Backend:
             else:
                 exe_wrapper = None
             if mesonlib.is_windows() or mesonlib.is_cygwin():
-                extra_paths = self.determine_windows_extra_paths(exe)
+                extra_paths = self.determine_windows_extra_paths(exe, [])
             else:
                 extra_paths = []
             cmd_args = []

mesonbuild/backend/ninjabackend.py

@@ -504,22 +504,30 @@ int dummy;
             # Add a dependency on all the outputs of this target
             for output in d.get_outputs():
                 elem.add_dep(os.path.join(self.get_target_dir(d), output))
+        serialize = False
+        extra_paths = []
         # If the target requires capturing stdout, then use the serialized
         # executable wrapper to capture that output and save it to a file.
-        #
+        if target.capture:
+            serialize = True
         # If the command line requires a newline, also use the wrapper, as
         # ninja does not support them in its build rule syntax.
-        #
+        if any('\n' in c for c in cmd):
+            serialize = True
         # Windows doesn't have -rpath, so for EXEs that need DLLs built within
         # the project, we need to set PATH so the DLLs are found. We use
         # a serialized executable wrapper for that and check if the
         # CustomTarget command needs extra paths first.
-        if (target.capture or any('\n' in c for c in cmd) or
+        if mesonlib.is_windows() or mesonlib.is_cygwin():
-                ((mesonlib.is_windows() or mesonlib.is_cygwin()) and
+            extra_bdeps = target.get_transitive_build_target_deps()
-                 self.determine_windows_extra_paths(target.command[0]))):
+            extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
+            if extra_paths:
+                serialize = True
+        if serialize:
             exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                                  # All targets are built from the build dir
                                                  self.environment.get_build_dir(),
+                                                 extra_paths=extra_paths,
                                                  capture=ofilenames[0] if target.capture else None)
             cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
             cmd_type = 'meson_exe.py custom'

mesonbuild/backend/vs2010backend.py

@@ -429,9 +429,12 @@ class Vs2010Backend(backends.Backend):
         # Always use a wrapper because MSBuild eats random characters when
         # there are many arguments.
         tdir_abs = os.path.join(self.environment.get_build_dir(), self.get_target_dir(target))
+        extra_bdeps = target.get_transitive_build_target_deps()
+        extra_paths = self.determine_windows_extra_paths(target.command[0], extra_bdeps)
         exe_data = self.serialize_executable(target.command[0], cmd[1:],
                                              # All targets run from the target dir
                                              tdir_abs,
+                                             extra_paths=extra_paths,
                                              capture=ofilenames[0] if target.capture else None)
         wrapper_cmd = self.environment.get_build_command() + ['--internal', 'exe', exe_data]
         ET.SubElement(customstep, 'Command').text = ' '.join(self.quote_arguments(wrapper_cmd))

mesonbuild/build.py

@@ -1549,6 +1549,24 @@ class CustomTarget(Target):
                 deps.append(c)
         return deps
 
+    def get_transitive_build_target_deps(self):
+        '''
+        Recursively fetch the build targets that this custom target depends on,
+        whether through `command:`, `depends:`, or `sources:` The recursion is
+        only performed on custom targets.
+        This is useful for setting PATH on Windows for finding required DLLs.
+        F.ex, if you have a python script that loads a C module that links to
+        other DLLs in your project.
+        '''
+        bdeps = set()
+        deps = self.get_target_dependencies()
+        for d in deps:
+            if isinstance(d, BuildTarget):
+                bdeps.add(d)
+            elif isinstance(d, CustomTarget):
+                bdeps.update(d.get_transitive_build_target_deps())
+        return bdeps
+
     def flatten_command(self, cmd):
         cmd = listify(cmd, unholder=True)
         final_cmd = []

mesonbuild/dependencies/misc.py

@@ -21,6 +21,8 @@ import shlex
 import shutil
 import sysconfig
 
+from pathlib import Path
+
 from .. import mlog
 from .. import mesonlib
 from ..mesonlib import Popen_safe, extract_as_list
@@ -603,6 +605,7 @@ class Python3Dependency(ExternalDependency):
     def __init__(self, environment, kwargs):
         super().__init__('python3', environment, None, kwargs)
         self.name = 'python3'
+        self.static = kwargs.get('static', False)
         # We can only be sure that it is Python 3 at this point
         self.version = '3'
         self.pkgdep = None
@@ -638,12 +641,55 @@ class Python3Dependency(ExternalDependency):
         else:
             mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.red('NO'))
 
+    @staticmethod
+    def get_windows_python_arch():
+        pyplat = sysconfig.get_platform()
+        if pyplat == 'mingw':
+            pycc = sysconfig.get_config_var('CC')
+            if pycc.startswith('x86_64'):
+                return '64'
+            elif pycc.startswith(('i686', 'i386')):
+                return '32'
+            else:
+                mlog.log('MinGW Python built with unknown CC {!r}, please file'
+                         'a bug'.format(pycc))
+                return None
+        elif pyplat == 'win32':
+            return '32'
+        elif pyplat in ('win64', 'win-amd64'):
+            return '64'
+        mlog.log('Unknown Windows Python platform {!r}'.format(pyplat))
+        return None
+
+    def get_windows_link_args(self):
+        pyplat = sysconfig.get_platform()
+        if pyplat.startswith('win'):
+            vernum = sysconfig.get_config_var('py_version_nodot')
+            if self.static:
+                libname = 'libpython{}.a'.format(vernum)
+            else:
+                libname = 'python{}.lib'.format(vernum)
+            lib = Path(sysconfig.get_config_var('base')) / 'libs' / libname
+        elif pyplat == 'mingw':
+            if self.static:
+                libname = sysconfig.get_config_var('LIBRARY')
+            else:
+                libname = sysconfig.get_config_var('LDLIBRARY')
+            lib = Path(sysconfig.get_config_var('LIBDIR')) / libname
+        if not lib.exists():
+            mlog.log('Could not find Python3 library {!r}'.format(str(lib)))
+            return None
+        return [str(lib)]
+
     def _find_libpy3_windows(self, env):
         '''
         Find python3 libraries on Windows and also verify that the arch matches
         what we are building for.
         '''
-        pyarch = sysconfig.get_platform()
+        pyarch = self.get_windows_python_arch()
+        if pyarch is None:
+            self.is_found = False
+            return
         arch = detect_cpu_family(env.coredata.compilers)
         if arch == 'x86':
             arch = '32'
@@ -656,22 +702,24 @@ class Python3Dependency(ExternalDependency):
             self.is_found = False
             return
         # Pyarch ends in '32' or '64'
-        if arch != pyarch[-2:]:
+        if arch != pyarch:
-            mlog.log('Need', mlog.bold(self.name),
+            mlog.log('Need', mlog.bold(self.name), 'for {}-bit, but '
-                     'for {}-bit, but found {}-bit'.format(arch, pyarch[-2:]))
+                     'found {}-bit'.format(arch, pyarch))
+            self.is_found = False
+            return
+        # This can fail if the library is not found
+        largs = self.get_windows_link_args()
+        if largs is None:
             self.is_found = False
             return
+        self.link_args = largs
+        # Compile args
         inc = sysconfig.get_path('include')
         platinc = sysconfig.get_path('platinclude')
         self.compile_args = ['-I' + inc]
         if inc != platinc:
             self.compile_args.append('-I' + platinc)
-        # Nothing exposes this directly that I coulf find
+        self.version = sysconfig.get_config_var('py_version')
-        basedir = sysconfig.get_config_var('base')
-        vernum = sysconfig.get_config_var('py_version_nodot')
-        self.link_args = ['-L{}/libs'.format(basedir),
-                          '-lpython{}'.format(vernum)]
-        self.version = sysconfig.get_config_var('py_version_short')
         self.is_found = True
 
     def get_methods(self):

test cases/python3/4 custom target depends extmodule/blaster.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+
+from pathlib import Path
+
+filedir = Path(os.path.dirname(__file__)).resolve()
+if list(filedir.glob('ext/*tachyon.*')):
+    sys.path.insert(0, (filedir / 'ext').as_posix())
+
+import tachyon
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-o', dest='output', default=None)
+
+options = parser.parse_args(sys.argv[1:])
+
+result = tachyon.phaserize('shoot')
+
+if options.output:
+    with open(options.output, 'w') as f:
+        f.write('success')
+
+if not isinstance(result, int):
+    print('Returned result not an integer.')
+    sys.exit(1)
+
+if result != 1:
+    print('Returned result {} is not 1.'.format(result))
+    sys.exit(1)

test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.c

@@ -0,0 +1,8 @@
+#ifdef _MSC_VER
+__declspec(dllexport)
+#endif
+const char*
+tachyon_phaser_command (void)
+{
+    return "shoot";
+}

test cases/python3/4 custom target depends extmodule/ext/lib/meson-tachyonlib.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#ifdef _MSC_VER
+__declspec(dllimport)
+#endif
+const char* tachyon_phaser_command (void);

test cases/python3/4 custom target depends extmodule/ext/lib/meson.build

@@ -0,0 +1,4 @@
+libtachyon = shared_library('tachyonlib', 'meson-tachyonlib.c')
+
+libtachyon_dep = declare_dependency(link_with : libtachyon,
+                                    include_directories : include_directories('.'))

test cases/python3/4 custom target depends extmodule/ext/meson.build

@@ -0,0 +1,6 @@
+subdir('lib')
+
+pylib = py3_mod.extension_module('tachyon',
+  'tachyon_module.c',
+  dependencies : [libtachyon_dep, py3_dep],
+)

test cases/python3/4 custom target depends extmodule/ext/tachyon_module.c

@@ -0,0 +1,51 @@
+/*
+  Copyright 2016 The Meson development team
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+/* A very simple Python extension module. */
+
+#include <Python.h>
+#include <string.h>
+
+#include "meson-tachyonlib.h"
+
+static PyObject* phaserize(PyObject *self, PyObject *args) {
+    const char *message;
+    int result;
+
+    if(!PyArg_ParseTuple(args, "s", &message))
+        return NULL;
+
+    result = strcmp(message, tachyon_phaser_command()) ? 0 : 1;
+    return PyLong_FromLong(result);
+}
+
+static PyMethodDef TachyonMethods[] = {
+    {"phaserize",  phaserize, METH_VARARGS,
+     "Shoot tachyon cannons."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef tachyonmodule = {
+   PyModuleDef_HEAD_INIT,
+   "tachyon",
+   NULL,
+   -1,
+   TachyonMethods
+};
+
+PyMODINIT_FUNC PyInit_tachyon(void) {
+    return PyModule_Create(&tachyonmodule);
+}

test cases/python3/4 custom target depends extmodule/meson.build

@@ -0,0 +1,35 @@
+project('Python extension module', 'c',
+  default_options : ['buildtype=release'])
+# Because Windows Python ships only with optimized libs,
+# we must build this project the same way.
+
+py3_mod = import('python3')
+py3 = py3_mod.find_python()
+py3_dep = dependency('python3', required : false)
+
+# Copy to the builddir so that blaster.py can find the built tachyon module
+# FIXME: We should automatically detect this case and append the correct paths
+# to PYTHONLIBDIR
+blaster_py = configure_file(input : 'blaster.py',
+                            output : 'blaster.py',
+                            configuration : configuration_data())
+
+check_exists = '''
+import os, sys
+with open(sys.argv[1], 'rb') as f:
+  assert(f.read() == b'success')
+'''
+if py3_dep.found()
+  subdir('ext')
+
+  out_txt = custom_target('tachyon flux',
+    input : blaster_py,
+    output : 'out.txt',
+    command : [py3, '@INPUT@', '-o', '@OUTPUT@'],
+    depends : pylib,
+    build_by_default: true)
+
+  test('flux', py3, args : ['-c', check_exists, out_txt])
+else
+  error('MESON_SKIP_TEST: Python3 libraries not found, skipping test.')
+endif