Python: Add 'limited_api' kwarg to extension_module

This commit adds a new keyword arg to extension_module() that enables
a user to target the Python Limited API, declaring the version of the
limited API that they wish to target.

Two new unittests have been added to test this functionality.

data/test.schema.json

@@ -26,9 +26,11 @@
               "exe",
               "shared_lib",
               "python_lib",
+              "python_limited_lib",
               "pdb",
               "implib",
               "py_implib",
+              "py_limited_implib",
               "implibempty",
               "expr"
             ]

docs/markdown/Builtin-options.md

@@ -370,12 +370,13 @@ install prefix. For example: if the install prefix is `/usr` and the
 
 ### Python module
 
-| Option           | Default value | Possible values             | Description |
+| Option            | Default value | Possible values             | Description |
-| ------           | ------------- | -----------------           | ----------- |
+| ------            | ------------- | -----------------           | ----------- |
-| bytecompile      | 0             | integer from -1 to 2        | What bytecode optimization level to use (Since 1.2.0) |
+| bytecompile       | 0             | integer from -1 to 2        | What bytecode optimization level to use (Since 1.2.0) |
-| install_env      | prefix        | {auto,prefix,system,venv}   | Which python environment to install to (Since 0.62.0) |
+| install_env       | prefix        | {auto,prefix,system,venv}   | Which python environment to install to (Since 0.62.0) |
-| platlibdir       |               | Directory path              | Directory for site-specific, platform-specific files (Since 0.60.0) |
+| platlibdir        |               | Directory path              | Directory for site-specific, platform-specific files (Since 0.60.0) |
-| purelibdir       |               | Directory path              | Directory for site-specific, non-platform-specific files  (Since 0.60.0) |
+| purelibdir        |               | Directory path              | Directory for site-specific, non-platform-specific files  (Since 0.60.0) |
+| allow_limited_api | true          | true, false                 | Disables project-wide use of the Python Limited API (Since 1.3.0) |
 
 *Since 0.60.0* The `python.platlibdir` and `python.purelibdir` options are used
 by the python module methods `python.install_sources()` and
@@ -405,3 +406,7 @@ python bytecode. Bytecode has 3 optimization levels:
 
 To this, Meson adds level `-1`, which is to not attempt to compile bytecode at
 all.
+
+*Since 1.3.0* The `python.allow_limited_api` option affects whether the
+`limited_api` keyword argument of the `extension_module` method is respected.
+If set to `false`, the effect of the `limited_api` argument is disabled.

docs/markdown/Python-module.md

@@ -101,6 +101,11 @@ the addition of the following:
   `/usr/lib/site-packages`. When subdir is passed to this method,
   it will be appended to that location. This keyword argument is
   mutually exclusive with `install_dir`
+- `limited_api`: *since 1.3.0* A string containing the Python version
+  of the [Py_LIMITED_API](https://docs.python.org/3/c-api/stable.html) that
+  the extension targets. For example, '3.7' to target Python 3.7's version of
+  the limited API. This behavior can be disabled by setting the value of
+  `python.allow_limited_api`. See [Python module options](Builtin-options.md#python-module).
 
 Additionally, the following diverge from [[shared_module]]'s default behavior:
 

docs/markdown/snippets/python_extension_module_limited_api.md

@@ -0,0 +1,5 @@
+## Support targeting Python's limited C API
+
+The Python module's `extension_module` function has gained the ability
+to build extensions which target Python's limited C API via a new keyword
+argument: `limited_api`.

mesonbuild/coredata.py

@@ -1300,6 +1300,8 @@ BUILTIN_CORE_OPTIONS: 'MutableKeyedOptionDictType' = OrderedDict([
      BuiltinOption(UserStringOption, 'Directory for site-specific, platform-specific files.', '')),
     (OptionKey('purelibdir', module='python'),
      BuiltinOption(UserStringOption, 'Directory for site-specific, non-platform-specific files.', '')),
+    (OptionKey('allow_limited_api', module='python'),
+     BuiltinOption(UserBooleanOption, 'Whether to allow use of the Python Limited API', True)),
 ])
 
 BUILTIN_OPTIONS = OrderedDict(chain(BUILTIN_DIR_OPTIONS.items(), BUILTIN_CORE_OPTIONS.items()))

mesonbuild/dependencies/python.py

@@ -44,6 +44,7 @@ if T.TYPE_CHECKING:
         paths: T.Dict[str, str]
         platform: str
         suffix: str
+        limited_api_suffix: str
         variables: T.Dict[str, str]
         version: str
 
@@ -94,6 +95,7 @@ class BasicPythonExternalProgram(ExternalProgram):
             'paths': {},
             'platform': 'sentinel',
             'suffix': 'sentinel',
+            'limited_api_suffix': 'sentinel',
             'variables': {},
             'version': '0.0',
         }
@@ -197,7 +199,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
         if self.link_libpython:
             # link args
             if mesonlib.is_windows():
-                self.find_libpy_windows(environment)
+                self.find_libpy_windows(environment, limited_api=False)
             else:
                 self.find_libpy(environment)
         else:
@@ -259,7 +261,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
         mlog.log(f'Unknown Windows Python platform {self.platform!r}')
         return None
 
-    def get_windows_link_args(self) -> T.Optional[T.List[str]]:
+    def get_windows_link_args(self, limited_api: bool) -> T.Optional[T.List[str]]:
         if self.platform.startswith('win'):
             vernum = self.variables.get('py_version_nodot')
             verdot = self.variables.get('py_version_short')
@@ -277,6 +279,8 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
                     else:
                         libpath = Path(f'python{vernum}.dll')
                 else:
+                    if limited_api:
+                        vernum = vernum[0]
                     libpath = Path('libs') / f'python{vernum}.lib'
                     # For a debug build, pyconfig.h may force linking with
                     # pythonX_d.lib (see meson#10776). This cannot be avoided
@@ -317,7 +321,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
             return None
         return [str(lib)]
 
-    def find_libpy_windows(self, env: 'Environment') -> None:
+    def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None:
         '''
         Find python3 libraries on Windows and also verify that the arch matches
         what we are building for.
@@ -332,7 +336,7 @@ class PythonSystemDependency(SystemDependency, _PythonDependencyBase):
             self.is_found = False
             return
         # This can fail if the library is not found
-        largs = self.get_windows_link_args()
+        largs = self.get_windows_link_args(limited_api)
         if largs is None:
             self.is_found = False
             return

mesonbuild/modules/python.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-import copy, json, os, shutil
+import copy, json, os, shutil, re
 import typing as T
 
 from . import ExtensionModule, ModuleInfo
@@ -32,7 +32,7 @@ from ..interpreterbase import (
     InvalidArguments, typed_pos_args, typed_kwargs, KwargInfo,
     FeatureNew, FeatureNewKwargs, disablerIfNotFound
 )
-from ..mesonlib import MachineChoice
+from ..mesonlib import MachineChoice, OptionKey
 from ..programs import ExternalProgram, NonExistingExternalProgram
 
 if T.TYPE_CHECKING:
@@ -65,7 +65,7 @@ if T.TYPE_CHECKING:
     MaybePythonProg = T.Union[NonExistingExternalProgram, 'PythonExternalProgram']
 
 
-mod_kwargs = {'subdir'}
+mod_kwargs = {'subdir', 'limited_api'}
 mod_kwargs.update(known_shmod_kwargs)
 mod_kwargs -= {'name_prefix', 'name_suffix'}
 
@@ -114,6 +114,7 @@ class PythonExternalProgram(BasicPythonExternalProgram):
 
 _PURE_KW = KwargInfo('pure', (bool, NoneType))
 _SUBDIR_KW = KwargInfo('subdir', str, default='')
+_LIMITED_API_KW = KwargInfo('limited_api', str, default='', since='1.3.0')
 _DEFAULTABLE_SUBDIR_KW = KwargInfo('subdir', (str, NoneType))
 
 class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
@@ -124,6 +125,7 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
         assert isinstance(prefix, str), 'for mypy'
         self.variables = info['variables']
         self.suffix = info['suffix']
+        self.limited_api_suffix = info['limited_api_suffix']
         self.paths = info['paths']
         self.pure = python.pure
         self.platlib_install_path = os.path.join(prefix, python.platlib)
@@ -148,7 +150,7 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
 
     @permittedKwargs(mod_kwargs)
     @typed_pos_args('python.extension_module', str, varargs=(str, mesonlib.File, CustomTarget, CustomTargetIndex, GeneratedList, StructuredSources, ExtractedObjects, BuildTarget))
-    @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, allow_unknown=True)
+    @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, _LIMITED_API_KW, allow_unknown=True)
     def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs: ExtensionModuleKw) -> 'SharedModule':
         if 'install_dir' in kwargs:
             if kwargs['subdir'] is not None:
@@ -161,9 +163,11 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
 
             kwargs['install_dir'] = self._get_install_dir_impl(False, subdir)
 
+        target_suffix = self.suffix
+
         new_deps = mesonlib.extract_as_list(kwargs, 'dependencies')
-        has_pydep = any(isinstance(dep, _PythonDependencyBase) for dep in new_deps)
+        pydep = next((dep for dep in new_deps if isinstance(dep, _PythonDependencyBase)), None)
-        if not has_pydep:
+        if pydep is None:
             pydep = self._dependency_method_impl({})
             if not pydep.found():
                 raise mesonlib.MesonException('Python dependency not found')
@@ -171,15 +175,62 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
             FeatureNew.single_use('python_installation.extension_module with implicit dependency on python',
                                   '0.63.0', self.subproject, 'use python_installation.dependency()',
                                   self.current_node)
+
+        limited_api_version = kwargs.pop('limited_api')
+        allow_limited_api = self.interpreter.environment.coredata.get_option(OptionKey('allow_limited_api', module='python'))
+        if limited_api_version != '' and allow_limited_api:
+
+            target_suffix = self.limited_api_suffix
+
+            limited_api_version_hex = self._convert_api_version_to_py_version_hex(limited_api_version, pydep.version)
+            limited_api_definition = f'-DPy_LIMITED_API={limited_api_version_hex}'
+
+            new_c_args = mesonlib.extract_as_list(kwargs, 'c_args')
+            new_c_args.append(limited_api_definition)
+            kwargs['c_args'] = new_c_args
+
+            new_cpp_args = mesonlib.extract_as_list(kwargs, 'cpp_args')
+            new_cpp_args.append(limited_api_definition)
+            kwargs['cpp_args'] = new_cpp_args
+
+            # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib
+            # into the linker path when not running in debug mode via a series #pragma comment(lib, "")
+            # directives. We manually override these here as this interferes with the intended
+            # use of the 'limited_api' kwarg
+            for_machine = self.interpreter.machine_from_native_kwarg(kwargs)
+            compilers = self.interpreter.environment.coredata.compilers[for_machine]
+            if any(compiler.get_id() == 'msvc' for compiler in compilers.values()):
+                pydep_copy = copy.copy(pydep)
+                pydep_copy.find_libpy_windows(self.env, limited_api=True)
+                if not pydep_copy.found():
+                    raise mesonlib.MesonException('Python dependency supporting limited API not found')
+
+                new_deps.remove(pydep)
+                new_deps.append(pydep_copy)
+
+                pyver = pydep.version.replace('.', '')
+                python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib'
+                python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib'
+
+                new_link_args = mesonlib.extract_as_list(kwargs, 'link_args')
+
+                is_debug = self.interpreter.environment.coredata.options[OptionKey('debug')].value
+                if is_debug:
+                    new_link_args.append(python_windows_debug_link_exception)
+                else:
+                    new_link_args.append(python_windows_release_link_exception)
+
+                kwargs['link_args'] = new_link_args
+
         kwargs['dependencies'] = new_deps
 
         # msys2's python3 has "-cpython-36m.dll", we have to be clever
         # FIXME: explain what the specific cleverness is here
-        split, suffix = self.suffix.rsplit('.', 1)
+        split, target_suffix = target_suffix.rsplit('.', 1)
         args = (args[0] + split, args[1])
 
         kwargs['name_prefix'] = ''
-        kwargs['name_suffix'] = suffix
+        kwargs['name_suffix'] = target_suffix
 
         if 'gnu_symbol_visibility' not in kwargs and \
                 (self.is_pypy or mesonlib.version_compare(self.version, '>=3.9')):
@@ -187,6 +238,22 @@ class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']):
 
         return self.interpreter.build_target(self.current_node, args, kwargs, SharedModule)
 
+    def _convert_api_version_to_py_version_hex(self, api_version: str, detected_version: str) -> str:
+        python_api_version_format = re.compile(r'[0-9]\.[0-9]{1,2}')
+        decimal_match = python_api_version_format.fullmatch(api_version)
+        if not decimal_match:
+            raise InvalidArguments(f'Python API version invalid: "{api_version}".')
+        if mesonlib.version_compare(api_version, '<3.2'):
+            raise InvalidArguments(f'Python Limited API version invalid: {api_version} (must be greater than 3.2)')
+        if mesonlib.version_compare(api_version, '>' + detected_version):
+            raise InvalidArguments(f'Python Limited API version too high: {api_version} (detected {detected_version})')
+
+        version_components = api_version.split('.')
+        major = int(version_components[0])
+        minor = int(version_components[1])
+
+        return '0x{:02x}{:02x}0000'.format(major, minor)
+
     def _dependency_method_impl(self, kwargs: TYPE_kwargs) -> Dependency:
         for_machine = self.interpreter.machine_from_native_kwarg(kwargs)
         identifier = get_dep_identifier(self._full_path(), kwargs)

mesonbuild/scripts/python_info.py

@@ -65,6 +65,20 @@ elif sys.version_info < (3, 8, 7):
 else:
     suffix = variables.get('EXT_SUFFIX')
 
+limited_api_suffix = None
+if sys.version_info >= (3, 2):
+    try:
+        from importlib.machinery import EXTENSION_SUFFIXES
+        limited_api_suffix = EXTENSION_SUFFIXES[1]
+    except Exception:
+        pass
+
+# pypy supports modules targetting the limited api but
+# does not use a special suffix to distinguish them:
+# https://doc.pypy.org/en/latest/cpython_differences.html#permitted-abi-tags-in-extensions
+if '__pypy__' in sys.builtin_module_names:
+    limited_api_suffix = suffix
+
 print(json.dumps({
   'variables': variables,
   'paths': paths,
@@ -76,4 +90,5 @@ print(json.dumps({
   'is_venv': sys.prefix != variables['base_prefix'],
   'link_libpython': links_against_libpython(),
   'suffix': suffix,
+  'limited_api_suffix': limited_api_suffix,
 }))

run_project_tests.py

@@ -148,7 +148,7 @@ class InstalledFile:
             canonical_compiler = 'msvc'
 
         python_suffix = python.info['suffix']
-
+        python_limited_suffix = python.info['limited_api_suffix']
         has_pdb = False
         if self.language in {'c', 'cpp'}:
             has_pdb = canonical_compiler == 'msvc'
@@ -167,7 +167,7 @@ class InstalledFile:
             return None
 
         # Handle the different types
-        if self.typ in {'py_implib', 'python_lib', 'python_file'}:
+        if self.typ in {'py_implib', 'py_limited_implib', 'python_lib', 'python_limited_lib', 'python_file'}:
             val = p.as_posix()
             val = val.replace('@PYTHON_PLATLIB@', python.platlib)
             val = val.replace('@PYTHON_PURELIB@', python.purelib)
@@ -176,6 +176,8 @@ class InstalledFile:
                 return p
             if self.typ == 'python_lib':
                 return p.with_suffix(python_suffix)
+            if self.typ == 'python_limited_lib':
+                return p.with_suffix(python_limited_suffix)
             if self.typ == 'py_implib':
                 p = p.with_suffix(python_suffix)
                 if env.machines.host.is_windows() and canonical_compiler == 'msvc':
@@ -184,6 +186,14 @@ class InstalledFile:
                     return p.with_suffix('.dll.a')
                 else:
                     return None
+            if self.typ == 'py_limited_implib':
+                p = p.with_suffix(python_limited_suffix)
+                if env.machines.host.is_windows() and canonical_compiler == 'msvc':
+                    return p.with_suffix('.lib')
+                elif env.machines.host.is_windows() or env.machines.host.is_cygwin():
+                    return p.with_suffix('.dll.a')
+                else:
+                    return None
         elif self.typ in {'file', 'dir'}:
             return p
         elif self.typ == 'shared_lib':

test cases/python/10 extmodule limited api disabled/meson.build

@@ -0,0 +1,10 @@
+project('Python limited api disabled', 'c',
+  default_options : ['buildtype=release', 'werror=true', 'python.allow_limited_api=false'])
+
+py_mod = import('python')
+py = py_mod.find_installation()
+
+module = py.extension_module('my_module',
+  'module.c',
+  limited_api: '3.7',
+)

test cases/python/10 extmodule limited api disabled/module.c

@@ -0,0 +1,17 @@
+#include <Python.h>
+
+#if defined(Py_LIMITED_API)
+#error "Py_LIMITED_API's definition by Meson should have been disabled."
+#endif
+
+static struct PyModuleDef my_module = {
+   PyModuleDef_HEAD_INIT,
+   "my_module",
+   NULL,
+   -1,
+   NULL
+};
+
+PyMODINIT_FUNC PyInit_my_module(void) {
+    return PyModule_Create(&my_module);
+}

test cases/python/9 extmodule limited api/limited.c

@@ -0,0 +1,19 @@
+#include <Python.h>
+
+#ifndef Py_LIMITED_API
+#error Py_LIMITED_API must be defined.
+#elif Py_LIMITED_API != 0x03070000
+#error Wrong value for Py_LIMITED_API
+#endif
+
+static struct PyModuleDef limited_module = {
+   PyModuleDef_HEAD_INIT,
+   "limited_api_test",
+   NULL,
+   -1,
+   NULL
+};
+
+PyMODINIT_FUNC PyInit_limited(void) {
+    return PyModule_Create(&limited_module);
+}

test cases/python/9 extmodule limited api/meson.build

@@ -0,0 +1,16 @@
+project('Python limited api', 'c',
+  default_options : ['buildtype=release', 'werror=true'])
+
+py_mod = import('python')
+py = py_mod.find_installation()
+
+ext_mod_limited = py.extension_module('limited',
+  'limited.c',
+  limited_api: '3.7',
+  install: true,
+)
+
+ext_mod = py.extension_module('not_limited',
+  'not_limited.c',
+  install: true,
+)

test cases/python/9 extmodule limited api/not_limited.c

@@ -0,0 +1,59 @@
+#include <Python.h>
+#include <stdio.h>
+
+#ifdef Py_LIMITED_API
+#error Py_LIMITED_API must not be defined.
+#endif
+
+/* This function explicitly calls functions whose declaration is elided when
+ * Py_LIMITED_API is defined. This is to test that the linker is actually
+ * linking to the right version of the library on Windows. */
+static PyObject *meth_not_limited(PyObject *self, PyObject *args)
+{
+    PyObject *list;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "o", &  list))
+        return NULL;
+
+    if (!PyList_Check(list)) {
+        PyErr_Format(PyExc_TypeError, "expected 'list'");
+        return NULL;
+    }
+
+    /* PyList_GET_SIZE and PyList_GET_ITEM are only available if Py_LIMITED_API
+     * is not defined. It seems likely that they will remain excluded from the
+     * limited API as their checked counterparts (PyList_GetSize and
+     * PyList_GetItem) are made available in that mode instead. */
+    size = PyList_GET_SIZE(list);
+    for(Py_ssize_t i = 0; i < size; ++i) {
+        PyObject *element = PyList_GET_ITEM(list, i);
+        if (element == NULL) {
+            return NULL;
+        }
+
+        if(PyObject_Print(element, stdout, Py_PRINT_RAW) == -1) {
+            return NULL;
+        }
+    }
+
+    Py_RETURN_NONE;
+}
+
+static struct PyMethodDef not_limited_methods[] = {
+    { "not_limited", meth_not_limited, METH_VARARGS,
+     "Calls functions whose declaration is elided by Py_LIMITED_API" },
+    { NULL, NULL, 0, NULL }
+};
+
+static struct PyModuleDef not_limited_module = {
+   PyModuleDef_HEAD_INIT,
+   "not_limited_api_test",
+   NULL,
+   -1,
+   not_limited_methods
+};
+
+PyMODINIT_FUNC PyInit_not_limited(void) {
+    return PyModule_Create(&not_limited_module);
+}

test cases/python/9 extmodule limited api/test.json

@@ -0,0 +1,8 @@
+{
+  "installed": [
+    {"type": "python_limited_lib", "file": "usr/@PYTHON_PLATLIB@/limited"},
+    {"type": "py_limited_implib", "file": "usr/@PYTHON_PLATLIB@/limited"},
+    {"type": "python_lib", "file": "usr/@PYTHON_PLATLIB@/not_limited"},
+    {"type": "py_implib", "file": "usr/@PYTHON_PLATLIB@/not_limited"}
+  ]
+}