From d1abdce88fa263b6e532901564e44ca510df1065 Mon Sep 17 00:00:00 2001 From: Andrew McNulty Date: Wed, 8 May 2024 14:51:25 +0200 Subject: [PATCH 1/5] Python: add load test to limited API test Based on the example in GH issue #13167, the limited API test has been extended with a test to load the compiled module to ensure it can be loaded correctly. --- .../python/9 extmodule limited api/limited.c | 14 ++++++++++++-- .../python/9 extmodule limited api/meson.build | 7 +++++++ .../python/9 extmodule limited api/test_limited.py | 6 ++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 test cases/python/9 extmodule limited api/test_limited.py diff --git a/test cases/python/9 extmodule limited api/limited.c b/test cases/python/9 extmodule limited api/limited.c index 0d1c71820..b977419ca 100644 --- a/test cases/python/9 extmodule limited api/limited.c +++ b/test cases/python/9 extmodule limited api/limited.c @@ -6,12 +6,22 @@ #error Wrong value for Py_LIMITED_API #endif +static PyObject * +hello(PyObject * Py_UNUSED(self), PyObject * Py_UNUSED(args)) { + return PyUnicode_FromString("hello world"); +} + +static struct PyMethodDef methods[] = { + { "hello", hello, METH_NOARGS, NULL }, + { NULL, NULL, 0, NULL }, +}; + static struct PyModuleDef limited_module = { PyModuleDef_HEAD_INIT, - "limited_api_test", + "limited", NULL, -1, - NULL + methods }; PyMODINIT_FUNC PyInit_limited(void) { diff --git a/test cases/python/9 extmodule limited api/meson.build b/test cases/python/9 extmodule limited api/meson.build index 68afc9699..bdf1b7b90 100644 --- a/test cases/python/9 extmodule limited api/meson.build +++ b/test cases/python/9 extmodule limited api/meson.build @@ -14,3 +14,10 @@ ext_mod = py.extension_module('not_limited', 'not_limited.c', install: true, ) + +test('load-test', + py, + args: [files('test_limited.py')], + env: { 'PYTHONPATH': meson.current_build_dir() }, + workdir: meson.current_source_dir() +) diff --git a/test cases/python/9 extmodule limited api/test_limited.py b/test cases/python/9 extmodule limited api/test_limited.py new file mode 100644 index 000000000..fcbf67b53 --- /dev/null +++ b/test cases/python/9 extmodule limited api/test_limited.py @@ -0,0 +1,6 @@ +from limited import hello + +def test_hello(): + assert hello() == "hello world" + +test_hello() From 4023bbfc3655cd9eba80021854ade9a0fcc97d11 Mon Sep 17 00:00:00 2001 From: Andrew McNulty Date: Wed, 8 May 2024 16:05:51 +0200 Subject: [PATCH 2/5] unittests: Add Python unittest for limited API This new unittest runs the existing Python limited API test case, and checks that the resulting library was linked to the correct library. --- unittests/pythontests.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/unittests/pythontests.py b/unittests/pythontests.py index 6079bd587..aaea906ea 100644 --- a/unittests/pythontests.py +++ b/unittests/pythontests.py @@ -11,7 +11,7 @@ from .allplatformstests import git_init from .baseplatformtests import BasePlatformTests from .helpers import * -from mesonbuild.mesonlib import MachineChoice, TemporaryDirectoryWinProof +from mesonbuild.mesonlib import MachineChoice, TemporaryDirectoryWinProof, is_windows from mesonbuild.modules.python import PythonModule class PythonTests(BasePlatformTests): @@ -86,3 +86,32 @@ python = pymod.find_installation('python3', required: true) if shutil.which('python2') or PythonModule._get_win_pythonpath('python2'): raise self.skipTest('python2 installed, already tested') self._test_bytecompile() + + def test_limited_api_linked_correct_lib(self): + if not is_windows(): + return self.skipTest('Test only run on Windows.') + + testdir = os.path.join(self.src_root, 'test cases', 'python', '9 extmodule limited api') + + self.init(testdir) + self.build() + + from importlib.machinery import EXTENSION_SUFFIXES + limited_suffix = EXTENSION_SUFFIXES[1] + + limited_library_path = os.path.join(self.builddir, f'limited{limited_suffix}') + self.assertPathExists(limited_library_path) + + limited_dep_name = 'python3.dll' + if shutil.which('dumpbin'): + # MSVC + output = subprocess.check_output(['dumpbin', '/DEPENDENTS', limited_library_path], + stderr=subprocess.STDOUT) + self.assertIn(limited_dep_name, output.decode()) + elif shutil.which('objdump'): + # mingw + output = subprocess.check_output(['objdump', '-p', limited_library_path], + stderr=subprocess.STDOUT) + self.assertIn(limited_dep_name, output.decode()) + else: + raise self.skipTest('Test needs either dumpbin(MSVC) or objdump(mingw).') From f66a527a7c4280c1652f2921912d149aeeca971d Mon Sep 17 00:00:00 2001 From: Andrew McNulty Date: Wed, 8 May 2024 19:01:29 +0200 Subject: [PATCH 3/5] Python: move Windows functions to dependency base This is in preparation for a future commit which makes it possible for a PythonPkgConfigDependency to be used in a context where previously only a PythonSystemDependency would be used. --- mesonbuild/dependencies/python.py | 125 +++++++++++++++--------------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py index a0a22c1a5..56dddd932 100644 --- a/mesonbuild/dependencies/python.py +++ b/mesonbuild/dependencies/python.py @@ -161,69 +161,6 @@ class _PythonDependencyBase(_Base): else: self.major_version = 2 - -class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase): - - def __init__(self, name: str, environment: 'Environment', - kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram', - libpc: bool = False): - if libpc: - mlog.debug(f'Searching for {name!r} via pkgconfig lookup in LIBPC') - else: - mlog.debug(f'Searching for {name!r} via fallback pkgconfig lookup in default paths') - - PkgConfigDependency.__init__(self, name, environment, kwargs) - _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) - - if libpc and not self.is_found: - mlog.debug(f'"python-{self.version}" could not be found in LIBPC, this is likely due to a relocated python installation') - - # pkg-config files are usually accurate starting with python 3.8 - if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'): - self.link_args = [] - - -class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase): - - def __init__(self, name: str, environment: 'Environment', - kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'): - ExtraFrameworkDependency.__init__(self, name, environment, kwargs) - _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) - - -class PythonSystemDependency(SystemDependency, _PythonDependencyBase): - - def __init__(self, name: str, environment: 'Environment', - kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'): - SystemDependency.__init__(self, name, environment, kwargs) - _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) - - # match pkg-config behavior - if self.link_libpython: - # link args - if mesonlib.is_windows(): - self.find_libpy_windows(environment, limited_api=False) - else: - self.find_libpy(environment) - else: - self.is_found = True - - # compile args - inc_paths = mesonlib.OrderedSet([ - self.variables.get('INCLUDEPY'), - self.paths.get('include'), - self.paths.get('platinclude')]) - - self.compile_args += ['-I' + path for path in inc_paths if path] - - # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/ - # https://github.com/python/cpython/pull/100137 - if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'): - self.compile_args += ['-DMS_WIN64='] - - if not self.clib_compiler.has_header('Python.h', '', environment, extra_args=self.compile_args)[0]: - self.is_found = False - def find_libpy(self, environment: 'Environment') -> None: if self.is_pypy: if self.major_version == 3: @@ -347,6 +284,68 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase): self.link_args = largs self.is_found = True +class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase): + + def __init__(self, name: str, environment: 'Environment', + kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram', + libpc: bool = False): + if libpc: + mlog.debug(f'Searching for {name!r} via pkgconfig lookup in LIBPC') + else: + mlog.debug(f'Searching for {name!r} via fallback pkgconfig lookup in default paths') + + PkgConfigDependency.__init__(self, name, environment, kwargs) + _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) + + if libpc and not self.is_found: + mlog.debug(f'"python-{self.version}" could not be found in LIBPC, this is likely due to a relocated python installation') + + # pkg-config files are usually accurate starting with python 3.8 + if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'): + self.link_args = [] + + +class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase): + + def __init__(self, name: str, environment: 'Environment', + kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'): + ExtraFrameworkDependency.__init__(self, name, environment, kwargs) + _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) + + +class PythonSystemDependency(SystemDependency, _PythonDependencyBase): + + def __init__(self, name: str, environment: 'Environment', + kwargs: T.Dict[str, T.Any], installation: 'BasicPythonExternalProgram'): + SystemDependency.__init__(self, name, environment, kwargs) + _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False)) + + # match pkg-config behavior + if self.link_libpython: + # link args + if mesonlib.is_windows(): + self.find_libpy_windows(environment, limited_api=False) + else: + self.find_libpy(environment) + else: + self.is_found = True + + # compile args + inc_paths = mesonlib.OrderedSet([ + self.variables.get('INCLUDEPY'), + self.paths.get('include'), + self.paths.get('platinclude')]) + + self.compile_args += ['-I' + path for path in inc_paths if path] + + # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/ + # https://github.com/python/cpython/pull/100137 + if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'): + self.compile_args += ['-DMS_WIN64='] + + if not self.clib_compiler.has_header('Python.h', '', environment, extra_args=self.compile_args)[0]: + self.is_found = False + @staticmethod def log_tried() -> str: return 'sysconfig' From fea7f94b6796e4ad296989c040d45ae3b4c3f444 Mon Sep 17 00:00:00 2001 From: Andrew McNulty Date: Wed, 8 May 2024 18:40:58 +0200 Subject: [PATCH 4/5] Python: fix limited API logic under mingw The Python Limited API support that was added in 1.2 had special handling of Windows, but the condition to check for Windows was not correct: it checked for MSVC and not for the target's OS. This causes mingw installations to not have the special handling applied. This commit fixes this to check explicitly for Windows. --- mesonbuild/modules/python.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py index d195a3fa5..e84bee187 100644 --- a/mesonbuild/modules/python.py +++ b/mesonbuild/modules/python.py @@ -184,13 +184,9 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): new_cpp_args.append(limited_api_definition) kwargs['cpp_args'] = new_cpp_args - # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib - # into the linker path when not running in debug mode via a series #pragma comment(lib, "") - # directives. We manually override these here as this interferes with the intended - # use of the 'limited_api' kwarg + # On Windows, the limited API DLL is python3.dll, not python3X.dll. for_machine = kwargs['native'] - compilers = self.interpreter.environment.coredata.compilers[for_machine] - if any(compiler.get_id() == 'msvc' for compiler in compilers.values()): + if self.interpreter.environment.machines[for_machine].is_windows(): pydep_copy = copy.copy(pydep) pydep_copy.find_libpy_windows(self.env, limited_api=True) if not pydep_copy.found(): @@ -199,6 +195,12 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): new_deps.remove(pydep) new_deps.append(pydep_copy) + # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib + # into the linker path when not running in debug mode via a series #pragma comment(lib, "") + # directives. We manually override these here as this interferes with the intended + # use of the 'limited_api' kwarg + compilers = self.interpreter.environment.coredata.compilers[for_machine] + if any(compiler.get_id() == 'msvc' for compiler in compilers.values()): pyver = pydep.version.replace('.', '') python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib' python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib' From 328011f77a1cef00655a697cd2e503a97754b745 Mon Sep 17 00:00:00 2001 From: Andrew McNulty Date: Wed, 8 May 2024 19:05:47 +0200 Subject: [PATCH 5/5] Python: link correct limited API lib on mingw This commit fixes GH issue #13167 by linking to the correct library under MINGW when the 'limited_api' kwarg is specified. --- mesonbuild/dependencies/python.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mesonbuild/dependencies/python.py b/mesonbuild/dependencies/python.py index 56dddd932..a74423cb7 100644 --- a/mesonbuild/dependencies/python.py +++ b/mesonbuild/dependencies/python.py @@ -248,9 +248,15 @@ class _PythonDependencyBase(_Base): lib = Path(self.variables.get('base_prefix')) / libpath elif self.platform.startswith('mingw'): if self.static: - libname = self.variables.get('LIBRARY') + if limited_api: + libname = self.variables.get('ABI3DLLLIBRARY') + else: + libname = self.variables.get('LIBRARY') else: - libname = self.variables.get('LDLIBRARY') + if limited_api: + libname = self.variables.get('ABI3LDLIBRARY') + else: + libname = self.variables.get('LDLIBRARY') lib = Path(self.variables.get('LIBDIR')) / libname else: raise mesonlib.MesonBugException(