Implement a generic python module

With contributions from Håvard Graff

ciimage/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get -y update && apt-get -y upgrade \
 && apt-get -y install gtk-sharp2 gtk-sharp2-gapi libglib2.0-cil-dev \
 && apt-get -y install libwmf-dev \
 && apt-get -y install qt4-linguist-tools qttools5-dev-tools \
+&& apt-get -y install python-dev \
 && python3 -m pip install hotdoc codecov
 
 ENV LANG='C.UTF-8'

mesonbuild/interpreter.py

@@ -1610,6 +1610,8 @@ class Interpreter(InterpreterBase):
             return DataHolder(item)
         elif isinstance(item, dependencies.InternalDependency):
             return InternalDependencyHolder(item)
+        elif isinstance(item, dependencies.ExternalDependency):
+            return DependencyHolder(item)
         elif isinstance(item, dependencies.ExternalProgram):
             return ExternalProgramHolder(item)
         elif hasattr(item, 'held_object'):

mesonbuild/modules/python.py

@@ -0,0 +1,432 @@
+# Copyright 2018 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+
+from pathlib import Path
+from .. import mesonlib
+from . import ExtensionModule
+from mesonbuild.modules import ModuleReturnValue
+from . import permittedSnippetKwargs
+from ..interpreterbase import (
+    noPosargs, noKwargs, permittedKwargs,
+    InterpreterObject, InvalidArguments
+)
+from ..interpreter import shlib_kwargs, ExternalProgramHolder
+from .. import mlog
+from ..environment import detect_cpu_family
+from ..dependencies.base import (
+    DependencyMethods, ExternalDependency,
+    ExternalProgram, PkgConfigDependency,
+    NonExistingExternalProgram
+)
+
+mod_kwargs = set(['subdir'])
+mod_kwargs.update(shlib_kwargs)
+mod_kwargs -= set(['install_dir', 'install_rpath'])
+
+
+def run_command(python, command):
+    _, stdout, _ = mesonlib.Popen_safe(python.get_command() + [
+        '-c',
+        command])
+
+    return stdout.strip()
+
+
+class PythonDependency(ExternalDependency):
+    def __init__(self, python_holder, environment, kwargs):
+        super().__init__('python', environment, None, kwargs)
+        self.name = 'python'
+        self.static = kwargs.get('static', False)
+        self.version = python_holder.version
+        self.platform = python_holder.platform
+        self.pkgdep = None
+        self.variables = python_holder.variables
+        self.paths = python_holder.paths
+        if mesonlib.version_compare(self.version, '>= 3.0'):
+            self.major_version = 3
+        else:
+            self.major_version = 2
+
+        if DependencyMethods.PKGCONFIG in self.methods:
+            pkg_version = self.variables.get('LDVERSION') or self.version
+            pkg_path = self.variables.get('LIBPC')
+            old_pkg_path = os.environ.get('PKG_CONFIG_PATH')
+
+            if pkg_path:
+                os.environ['PKG_CONFIG_PATH'] = pkg_path
+
+            try:
+                self.pkgdep = PkgConfigDependency('python-{}'.format(pkg_version), environment, kwargs)
+                if self.pkgdep.found():
+                    self.compile_args = self.pkgdep.get_compile_args()
+                    self.link_args = self.pkgdep.get_link_args()
+                    self.is_found = True
+                    self.pcdep = self.pkgdep
+                    if old_pkg_path:
+                        os.environ['PKG_CONFIG_PATH'] = old_pkg_path
+                    else:
+                        os.environ.pop('PKG_CONFIG_PATH', None)
+                    return
+                else:
+                    self.pkgdep = None
+            except Exception:
+                pass
+
+            if old_pkg_path:
+                os.environ['PKG_CONFIG_PATH'] = old_pkg_path
+            else:
+                os.environ.pop('PKG_CONFIG_PATH', None)
+
+        if not self.is_found:
+            if mesonlib.is_windows() and DependencyMethods.SYSCONFIG in self.methods:
+                self._find_libpy_windows(environment)
+
+        if self.is_found:
+            mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.green('YES'))
+        else:
+            mlog.log('Dependency', mlog.bold(self.name), 'found:', mlog.red('NO'))
+
+    def get_windows_python_arch(self):
+        if self.platform == 'mingw':
+            pycc = self.variables.get('CC')
+            if pycc.startswith('x86_64'):
+                return '64'
+            elif pycc.startswith(('i686', 'i386')):
+                return '32'
+            else:
+                mlog.log('MinGW Python built with unknown CC {!r}, please file'
+                         'a bug'.format(pycc))
+                return None
+        elif self.platform == 'win32':
+            return '32'
+        elif self.platform in ('win64', 'win-amd64'):
+            return '64'
+        mlog.log('Unknown Windows Python platform {!r}'.format(self.platform))
+        return None
+
+    def get_windows_link_args(self):
+        if self.platform.startswith('win'):
+            vernum = self.variables.get('py_version_nodot')
+            if self.static:
+                libname = 'libpython{}.a'.format(vernum)
+            else:
+                libname = 'python{}.lib'.format(vernum)
+            lib = Path(self.variables.get('base')) / 'libs' / libname
+        elif self.platform == 'mingw':
+            if self.static:
+                libname = self.variables.get('LIBRARY')
+            else:
+                libname = self.variables.get('LDLIBRARY')
+            lib = Path(self.variables.get('LIBDIR')) / libname
+        if not lib.exists():
+            mlog.log('Could not find Python3 library {!r}'.format(str(lib)))
+            return None
+        return [str(lib)]
+
+    def _find_libpy_windows(self, env):
+        '''
+        Find python3 libraries on Windows and also verify that the arch matches
+        what we are building for.
+        '''
+        pyarch = self.get_windows_python_arch()
+        if pyarch is None:
+            self.is_found = False
+            return
+        arch = detect_cpu_family(env.coredata.compilers)
+        if arch == 'x86':
+            arch = '32'
+        elif arch == 'x86_64':
+            arch = '64'
+        else:
+            # We can't cross-compile Python 3 dependencies on Windows yet
+            mlog.log('Unknown architecture {!r} for'.format(arch),
+                     mlog.bold(self.name))
+            self.is_found = False
+            return
+        # Pyarch ends in '32' or '64'
+        if arch != pyarch:
+            mlog.log('Need', mlog.bold(self.name), 'for {}-bit, but '
+                     'found {}-bit'.format(arch, pyarch))
+            self.is_found = False
+            return
+        # This can fail if the library is not found
+        largs = self.get_windows_link_args()
+        if largs is None:
+            self.is_found = False
+            return
+        self.link_args = largs
+        # Compile args
+        inc_paths = mesonlib.OrderedSet([
+            self.variables.get('INCLUDEPY'),
+            self.paths.get('include'),
+            self.paths.get('platinclude')])
+
+        self.compile_args += ['-I' + path for path in inc_paths if path]
+
+        # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/
+        if pyarch == '64' and self.major_version == 2:
+            self.compile_args += ['-DMS_WIN64']
+
+        self.is_found = True
+
+    @staticmethod
+    def get_methods():
+        if mesonlib.is_windows():
+            return [DependencyMethods.PKGCONFIG, DependencyMethods.SYSCONFIG]
+        elif mesonlib.is_osx():
+            return [DependencyMethods.PKGCONFIG, DependencyMethods.EXTRAFRAMEWORK]
+        else:
+            return [DependencyMethods.PKGCONFIG]
+
+    def get_pkgconfig_variable(self, variable_name, kwargs):
+        if self.pkgdep:
+            return self.pkgdep.get_pkgconfig_variable(variable_name, kwargs)
+        else:
+            return super().get_pkgconfig_variable(variable_name, kwargs)
+
+
+VARIABLES_COMMAND = '''
+import sysconfig
+import json
+
+print (json.dumps (sysconfig.get_config_vars()))
+'''
+
+
+PATHS_COMMAND = '''
+import sysconfig
+import json
+
+print (json.dumps(sysconfig.get_paths()))
+'''
+
+
+INSTALL_PATHS_COMMAND = '''
+import sysconfig
+import json
+
+print (json.dumps(sysconfig.get_paths(scheme='posix_prefix', vars={'base': '', 'platbase': '', 'installed_base': ''})))
+'''
+
+
+class PythonHolder(ExternalProgramHolder, InterpreterObject):
+    def __init__(self, interpreter, python):
+        InterpreterObject.__init__(self)
+        ExternalProgramHolder.__init__(self, python)
+        self.interpreter = interpreter
+        prefix = self.interpreter.environment.coredata.get_builtin_option('prefix')
+        self.variables = json.loads(run_command(python, VARIABLES_COMMAND))
+        self.paths = json.loads(run_command(python, PATHS_COMMAND))
+        install_paths = json.loads(run_command(python, INSTALL_PATHS_COMMAND))
+        self.platlib_install_path = os.path.join(prefix, install_paths['platlib'][1:])
+        self.purelib_install_path = os.path.join(prefix, install_paths['purelib'][1:])
+        self.version = run_command(python, "import sysconfig; print (sysconfig.get_python_version())")
+        self.platform = run_command(python, "import sysconfig; print (sysconfig.get_platform())")
+
+    @permittedSnippetKwargs(mod_kwargs)
+    def extension_module(self, interpreter, state, args, kwargs):
+        if 'name_prefix' in kwargs:
+            raise mesonlib.MesonException('Name_prefix is set automatically, specifying it is forbidden.')
+        if 'name_suffix' in kwargs:
+            raise mesonlib.MesonException('Name_suffix is set automatically, specifying it is forbidden.')
+
+        subdir = kwargs.pop('subdir', '')
+        if not isinstance(subdir, str):
+            raise InvalidArguments('"subdir" argument must be a string.')
+
+        kwargs['install_dir'] = os.path.join(self.platlib_install_path, subdir)
+
+        suffix = self.variables.get('EXT_SUFFIX') or self.variables.get('SO') or self.variables.get('.so')
+
+        # msys2's python3 has "-cpython-36m.dll", we have to be clever
+        split = suffix.rsplit('.', 1)
+        suffix = split.pop(-1)
+        args[0] += ''.join(s for s in split)
+
+        kwargs['name_prefix'] = ''
+        kwargs['name_suffix'] = suffix
+
+        return interpreter.func_shared_module(None, args, kwargs)
+
+    def dependency(self, interpreter, state, args, kwargs):
+        dep = PythonDependency(self, interpreter.environment, kwargs)
+        return interpreter.holderify(dep)
+
+    @permittedSnippetKwargs(['pure', 'subdir'])
+    def install_sources(self, interpreter, state, args, kwargs):
+        pure = kwargs.pop('pure', False)
+        if not isinstance(pure, bool):
+            raise InvalidArguments('"pure" argument must be a boolean.')
+
+        subdir = kwargs.pop('subdir', '')
+        if not isinstance(subdir, str):
+            raise InvalidArguments('"subdir" argument must be a string.')
+
+        if pure:
+            kwargs['install_dir'] = os.path.join(self.purelib_install_path, subdir)
+        else:
+            kwargs['install_dir'] = os.path.join(self.platlib_install_path, subdir)
+
+        return interpreter.func_install_data(None, args, kwargs)
+
+    @noPosargs
+    @permittedKwargs(['pure', 'subdir'])
+    def get_install_dir(self, node, args, kwargs):
+        pure = kwargs.pop('pure', False)
+        if not isinstance(pure, bool):
+            raise InvalidArguments('"pure" argument must be a boolean.')
+
+        subdir = kwargs.pop('subdir', '')
+        if not isinstance(subdir, str):
+            raise InvalidArguments('"subdir" argument must be a string.')
+
+        if pure:
+            res = os.path.join(self.purelib_install_path, subdir)
+        else:
+            res = os.path.join(self.platlib_install_path, subdir)
+
+        return ModuleReturnValue(res, [])
+
+    @noPosargs
+    @noKwargs
+    def language_version(self, node, args, kwargs):
+        return ModuleReturnValue(self.version, [])
+
+    @noPosargs
+    @noKwargs
+    def found(self, node, args, kwargs):
+        return ModuleReturnValue(True, [])
+
+    @noKwargs
+    def get_path(self, node, args, kwargs):
+        if len(args) != 1:
+            raise InvalidArguments('get_path takes exactly one positional argument.')
+        path_name = args[0]
+        if not isinstance(path_name, str):
+            raise InvalidArguments('get_path argument must be a string.')
+
+        path = self.paths.get(path_name)
+
+        if path is None:
+            raise InvalidArguments('{} is not a valid path name'.format(path_name))
+
+        return ModuleReturnValue(path, [])
+
+    @noKwargs
+    def get_variable(self, node, args, kwargs):
+        if len(args) != 1:
+            raise InvalidArguments('get_variable takes exactly one positional argument.')
+        var_name = args[0]
+        if not isinstance(var_name, str):
+            raise InvalidArguments('get_variable argument must be a string.')
+
+        var = self.variables.get(var_name)
+
+        if var is None:
+            raise InvalidArguments('{} is not a valid path name'.format(var_name))
+
+        return ModuleReturnValue(var, [])
+
+    def method_call(self, method_name, args, kwargs):
+        try:
+            fn = getattr(self, method_name)
+        except AttributeError:
+            raise InvalidArguments('Python object does not have method %s.' % method_name)
+
+        if method_name in ['extension_module', 'dependency', 'install_sources']:
+            value = fn(self.interpreter, None, args, kwargs)
+            return self.interpreter.holderify(value)
+        elif method_name in ['get_variable', 'get_path', 'found', 'language_version', 'get_install_dir']:
+            value = fn(None, args, kwargs)
+            return self.interpreter.module_method_callback(value)
+        else:
+            raise InvalidArguments('Python object does not have method %s.' % method_name)
+
+
+class PythonModule(ExtensionModule):
+    def __init__(self):
+        super().__init__()
+        self.snippets.add('find')
+
+    # https://www.python.org/dev/peps/pep-0397/
+    def _get_win_pythonpath(self, name_or_path):
+        if name_or_path not in ['python2', 'python3']:
+            return None
+        ver = {'python2': '-2', 'python3': '-3'}[name_or_path]
+        cmd = ['py', ver, '-c', "import sysconfig; print(sysconfig.get_config_var('BINDIR'))"]
+        _, stdout, _ = mesonlib.Popen_safe(cmd)
+        dir = stdout.strip()
+        if os.path.exists(dir):
+            return os.path.join(dir, 'python')
+        else:
+            return None
+
+    @permittedSnippetKwargs(['required'])
+    def find(self, interpreter, state, args, kwargs):
+        required = kwargs.get('required', True)
+        if not isinstance(required, bool):
+            raise InvalidArguments('"required" argument must be a boolean.')
+
+        if len(args) != 1:
+            raise InvalidArguments('find takes zero or one positional argument.')
+
+        if args:
+            name_or_path = args[0]
+            if not isinstance(name_or_path, str):
+                raise InvalidArguments('find argument must be a string.')
+        else:
+            name_or_path = None
+
+        if not name_or_path:
+            mlog.log("Using meson's python {}".format(mesonlib.python_command))
+            python = ExternalProgram('python3', mesonlib.python_command, silent=True)
+        else:
+            if mesonlib.is_windows():
+                pythonpath = self._get_win_pythonpath(name_or_path)
+                if pythonpath is not None:
+                    name_or_path = pythonpath
+            python = ExternalProgram(name_or_path, silent = True)
+            # Last ditch effort, python2 or python3 can be named python
+            # on various platforms, let's not give up just yet, if an executable
+            # named python is available and has a compatible version, let's use
+            # it
+            if not python.found() and name_or_path in ['python2', 'python3']:
+                python = ExternalProgram('python', silent = True)
+                if python.found():
+                    version = run_command(python, "import sysconfig; print (sysconfig.get_python_version())")
+                    if not version or \
+                            name_or_path == 'python2' and mesonlib.version_compare(version, '>= 3.0') or \
+                            name_or_path == 'python3' and not mesonlib.version_compare(version, '>= 3.0'):
+                        res = NonExistingExternalProgram()
+
+        if not python.found():
+            if required:
+                raise mesonlib.MesonException('{} not found'.format(name_or_path or 'python'))
+            res = ExternalProgramHolder(NonExistingExternalProgram())
+        else:
+            # Sanity check, we expect to have something that at least quacks in tune
+            version = run_command(python, "import sysconfig; print (sysconfig.get_python_version())")
+            if not version:
+                res = ExternalProgramHolder(NonExistingExternalProgram())
+            else:
+                res = PythonHolder(interpreter, python)
+
+        return res
+
+
+def initialize():
+    return PythonModule()

run_unittests.py

@@ -2838,6 +2838,43 @@ class LinuxArmCrossCompileTests(BasePlatformTests):
         self.assertNotIn('-DBUILD_ENVIRONMENT_ONLY', compdb[0]['command'])
 
 
+class PythonTests(BasePlatformTests):
+    '''
+    Tests that verify compilation of python extension modules
+    '''
+    def test_versions(self):
+        if self.backend is not Backend.ninja:
+            raise unittest.SkipTest('Skipping python tests with {} backend'.format(self.backend.name))
+
+        testdir = os.path.join(self.src_root, 'test cases', 'python', '1 extmodule')
+
+        # No python version specified, this will use meson's python
+        self.init(testdir)
+        self.build()
+        self.run_tests()
+        self.wipe()
+
+        # When specifying a known name, (python2 / python3) the module
+        # will also try 'python' as a fallback and use it if the major
+        # version matches
+        self.init(testdir, ['-Dpython=python2'])
+        self.build()
+        self.run_tests()
+        self.wipe()
+
+        # The test is configured to error out with MESON_SKIP_TEST
+        # in case it could not find python
+        with self.assertRaises(unittest.SkipTest):
+            self.init(testdir, ['-Dpython=not-python'])
+        self.wipe()
+
+        # While dir is an external command on both Windows and Linux,
+        # it certainly isn't python
+        with self.assertRaises(unittest.SkipTest):
+            self.init(testdir, ['-Dpython=dir'])
+        self.wipe()
+
+
 class RewriterTests(unittest.TestCase):
 
     def setUp(self):
@@ -2912,7 +2949,7 @@ def unset_envs():
 
 if __name__ == '__main__':
     unset_envs()
-    cases = ['InternalTests', 'AllPlatformTests', 'FailureTests']
+    cases = ['InternalTests', 'AllPlatformTests', 'FailureTests', 'PythonTests']
     if not is_windows():
         cases += ['LinuxlikeTests']
         if should_run_linux_cross_tests():

test cases/python/1 extmodule/blaster.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+import sys
+import tachyon
+
+result = tachyon.phaserize('shoot')
+
+if not isinstance(result, int):
+    print('Returned result not an integer.')
+    sys.exit(1)
+
+if result != 1:
+    print('Returned result {} is not 1.'.format(result))
+    sys.exit(1)

test cases/python/1 extmodule/ext/meson.build

@@ -0,0 +1,6 @@
+pylib = py.extension_module('tachyon',
+  'tachyon_module.c',
+  dependencies : py_dep,
+)
+
+pypathdir = meson.current_build_dir()

test cases/python/1 extmodule/ext/tachyon_module.c

@@ -0,0 +1,59 @@
+/*
+  Copyright 2018 The Meson development team
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+/* A very simple Python extension module. */
+
+#include <Python.h>
+#include <string.h>
+
+static PyObject* phaserize(PyObject *self, PyObject *args) {
+    const char *message;
+    int result;
+
+    if(!PyArg_ParseTuple(args, "s", &message))
+        return NULL;
+
+    result = strcmp(message, "shoot") ? 0 : 1;
+#if PY_VERSION_HEX < 0x03000000
+    return PyInt_FromLong(result);
+#else
+    return PyLong_FromLong(result);
+#endif
+}
+
+static PyMethodDef TachyonMethods[] = {
+    {"phaserize",  phaserize, METH_VARARGS,
+     "Shoot tachyon cannons."},
+    {NULL, NULL, 0, NULL}
+};
+
+#if PY_VERSION_HEX < 0x03000000
+PyMODINIT_FUNC inittachyon(void) {
+    Py_InitModule("tachyon", TachyonMethods);
+}
+#else
+static struct PyModuleDef tachyonmodule = {
+   PyModuleDef_HEAD_INIT,
+   "tachyon",
+   NULL,
+   -1,
+   TachyonMethods
+};
+
+PyMODINIT_FUNC PyInit_tachyon(void) {
+    return PyModule_Create(&tachyonmodule);
+}
+#endif

test cases/python/1 extmodule/meson.build

@@ -0,0 +1,23 @@
+project('Python extension module', 'c',
+  default_options : ['buildtype=release'])
+
+py_mod = import('python')
+
+py = py_mod.find(get_option('python'), required : false)
+
+if py.found()
+  py_dep = py.dependency()
+
+  if py_dep.found()
+    subdir('ext')
+
+    test('extmod',
+      py,
+      args : files('blaster.py'),
+      env : ['PYTHONPATH=' + pypathdir])
+  else
+    error('MESON_SKIP_TEST: Python libraries not found, skipping test.')
+  endif
+else
+  error('MESON_SKIP_TEST: Python not found, skipping test.')
+endif

test cases/python/1 extmodule/meson_options.txt

@@ -0,0 +1,3 @@
+option('python', type: 'string',
+  description: 'Name of or path to the python executable'
+)