Merge pull request #13171 from amcn/mingw-python-limited-api

Python: Fix limited API under mingw

mesonbuild/dependencies/python.py

@@ -161,69 +161,6 @@ class _PythonDependencyBase(_Base):
         else:
             self.major_version = 2
 
-
-class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase):
-
-    def __init__(self, name: str, environment: 'Environment',
-                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram',
-                 libpc: bool = False):
-        if libpc:
-            mlog.debug(f'Searching for {name!r} via pkgconfig lookup in LIBPC')
-        else:
-            mlog.debug(f'Searching for {name!r} via fallback pkgconfig lookup in default paths')
-
-        PkgConfigDependency.__init__(self, name, environment, kwargs)
-        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
-
-        if libpc and not self.is_found:
-            mlog.debug(f'"python-{self.version}" could not be found in LIBPC, this is likely due to a relocated python installation')
-
-        # pkg-config files are usually accurate starting with python 3.8
-        if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'):
-            self.link_args = []
-
-
-class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase):
-
-    def __init__(self, name: str, environment: 'Environment',
-                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'):
-        ExtraFrameworkDependency.__init__(self, name, environment, kwargs)
-        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
-
-
-class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
-
-    def __init__(self, name: str, environment: 'Environment',
-                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'):
-        SystemDependency.__init__(self, name, environment, kwargs)
-        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
-
-        # match pkg-config behavior
-        if self.link_libpython:
-            # link args
-            if mesonlib.is_windows():
-                self.find_libpy_windows(environment, limited_api=False)
-            else:
-                self.find_libpy(environment)
-        else:
-            self.is_found = True
-
-        # compile args
-        inc_paths = mesonlib.OrderedSet([
-            self.variables.get('INCLUDEPY'),
-            self.paths.get('include'),
-            self.paths.get('platinclude')])
-
-        self.compile_args += ['-I' + path for path in inc_paths if path]
-
-        # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/
-        # https://github.com/python/cpython/pull/100137
-        if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'):
-            self.compile_args += ['-DMS_WIN64=']
-
-        if not self.clib_compiler.has_header('Python.h', '', environment, extra_args=self.compile_args)[0]:
-            self.is_found = False
-
     def find_libpy(self, environment: 'Environment') -> None:
         if self.is_pypy:
             if self.major_version == 3:
@@ -311,9 +248,15 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
             lib = Path(self.variables.get('base_prefix')) / libpath
         elif self.platform.startswith('mingw'):
             if self.static:
-                libname = self.variables.get('LIBRARY')
+                if limited_api:
+                    libname = self.variables.get('ABI3DLLLIBRARY')
+                else:
+                    libname = self.variables.get('LIBRARY')
             else:
-                libname = self.variables.get('LDLIBRARY')
+                if limited_api:
+                    libname = self.variables.get('ABI3LDLIBRARY')
+                else:
+                    libname = self.variables.get('LDLIBRARY')
             lib = Path(self.variables.get('LIBDIR')) / libname
         else:
             raise mesonlib.MesonBugException(
@@ -347,6 +290,68 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
         self.link_args = largs
         self.is_found = True
 
+class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase):
+
+    def __init__(self, name: str, environment: 'Environment',
+                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram',
+                 libpc: bool = False):
+        if libpc:
+            mlog.debug(f'Searching for {name!r} via pkgconfig lookup in LIBPC')
+        else:
+            mlog.debug(f'Searching for {name!r} via fallback pkgconfig lookup in default paths')
+
+        PkgConfigDependency.__init__(self, name, environment, kwargs)
+        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
+
+        if libpc and not self.is_found:
+            mlog.debug(f'"python-{self.version}" could not be found in LIBPC, this is likely due to a relocated python installation')
+
+        # pkg-config files are usually accurate starting with python 3.8
+        if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'):
+            self.link_args = []
+
+
+class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase):
+
+    def __init__(self, name: str, environment: 'Environment',
+                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'):
+        ExtraFrameworkDependency.__init__(self, name, environment, kwargs)
+        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
+
+
+class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
+
+    def __init__(self, name: str, environment: 'Environment',
+                 kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'):
+        SystemDependency.__init__(self, name, environment, kwargs)
+        _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False))
+
+        # match pkg-config behavior
+        if self.link_libpython:
+            # link args
+            if mesonlib.is_windows():
+                self.find_libpy_windows(environment, limited_api=False)
+            else:
+                self.find_libpy(environment)
+        else:
+            self.is_found = True
+
+        # compile args
+        inc_paths = mesonlib.OrderedSet([
+            self.variables.get('INCLUDEPY'),
+            self.paths.get('include'),
+            self.paths.get('platinclude')])
+
+        self.compile_args += ['-I' + path for path in inc_paths if path]
+
+        # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/
+        # https://github.com/python/cpython/pull/100137
+        if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'):
+            self.compile_args += ['-DMS_WIN64=']
+
+        if not self.clib_compiler.has_header('Python.h', '', environment, extra_args=self.compile_args)[0]:
+            self.is_found = False
+
     @staticmethod
     def log_tried() -> str:
         return 'sysconfig'

mesonbuild/modules/python.py

@@ -184,13 +184,9 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
             new_cpp_args.append(limited_api_definition)
             kwargs['cpp_args'] = new_cpp_args
 
-            # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib
-            # into the linker path when not running in debug mode via a series #pragma comment(lib, "")
-            # directives. We manually override these here as this interferes with the intended
-            # use of the 'limited_api' kwarg
+            # On Windows, the limited API DLL is python3.dll, not python3X.dll.
             for_machine = kwargs['native']
-            compilers = self.interpreter.environment.coredata.compilers[for_machine]
-            if any(compiler.get_id() == 'msvc' for compiler in compilers.values()):
+            if self.interpreter.environment.machines[for_machine].is_windows():
                 pydep_copy = copy.copy(pydep)
                 pydep_copy.find_libpy_windows(self.env, limited_api=True)
                 if not pydep_copy.found():
@@ -199,6 +195,12 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
                 new_deps.remove(pydep)
                 new_deps.append(pydep_copy)
 
+            # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib
+            # into the linker path when not running in debug mode via a series #pragma comment(lib, "")
+            # directives. We manually override these here as this interferes with the intended
+            # use of the 'limited_api' kwarg
+            compilers = self.interpreter.environment.coredata.compilers[for_machine]
+            if any(compiler.get_id() == 'msvc' for compiler in compilers.values()):
                 pyver = pydep.version.replace('.', '')
                 python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib'
                 python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib'

test cases/python/9 extmodule limited api/limited.c

@@ -6,12 +6,22 @@
 #error Wrong value for Py_LIMITED_API
 #endif
 
+static PyObject *
+hello(PyObject * Py_UNUSED(self), PyObject * Py_UNUSED(args)) {
+    return PyUnicode_FromString("hello world");
+}
+
+static struct PyMethodDef methods[] = {
+    { "hello", hello, METH_NOARGS, NULL },
+    { NULL, NULL, 0, NULL },
+};
+
 static struct PyModuleDef limited_module = {
    PyModuleDef_HEAD_INIT,
-   "limited_api_test",
+   "limited",
    NULL,
    -1,
-   NULL
+   methods
 };
 
 PyMODINIT_FUNC PyInit_limited(void) {

test cases/python/9 extmodule limited api/meson.build

@@ -14,3 +14,10 @@ ext_mod = py.extension_module('not_limited',
   'not_limited.c',
   install: true,
 )
+
+test('load-test',
+  py,
+  args: [files('test_limited.py')],
+  env: { 'PYTHONPATH': meson.current_build_dir() },
+  workdir: meson.current_source_dir()
+)

test cases/python/9 extmodule limited api/test_limited.py

@@ -0,0 +1,6 @@
+from limited import hello
+
+def test_hello():
+    assert hello() == "hello world"
+
+test_hello()

unittests/pythontests.py

@@ -11,7 +11,7 @@ from .allplatformstests import git_init
 from .baseplatformtests import BasePlatformTests
 from .helpers import *
 
-from mesonbuild.mesonlib import MachineChoice, TemporaryDirectoryWinProof
+from mesonbuild.mesonlib import MachineChoice, TemporaryDirectoryWinProof, is_windows
 from mesonbuild.modules.python import PythonModule
 
 class PythonTests(BasePlatformTests):
@@ -86,3 +86,32 @@ python = pymod.find_installation('python3', required: true)
         if shutil.which('python2') or PythonModule._get_win_pythonpath('python2'):
             raise self.skipTest('python2 installed, already tested')
         self._test_bytecompile()
+
+    def test_limited_api_linked_correct_lib(self):
+        if not is_windows():
+            return self.skipTest('Test only run on Windows.')
+
+        testdir = os.path.join(self.src_root, 'test cases', 'python', '9 extmodule limited api')
+
+        self.init(testdir)
+        self.build()
+
+        from importlib.machinery import EXTENSION_SUFFIXES
+        limited_suffix = EXTENSION_SUFFIXES[1]
+
+        limited_library_path = os.path.join(self.builddir, f'limited{limited_suffix}')
+        self.assertPathExists(limited_library_path)
+
+        limited_dep_name = 'python3.dll'
+        if shutil.which('dumpbin'):
+            # MSVC
+            output = subprocess.check_output(['dumpbin', '/DEPENDENTS', limited_library_path],
+                                            stderr=subprocess.STDOUT)
+            self.assertIn(limited_dep_name, output.decode())
+        elif shutil.which('objdump'):
+            # mingw
+            output = subprocess.check_output(['objdump', '-p', limited_library_path],
+                                             stderr=subprocess.STDOUT)
+            self.assertIn(limited_dep_name, output.decode())
+        else:
+            raise self.skipTest('Test needs either dumpbin(MSVC) or objdump(mingw).')