devenv: Set PYTHONPATH where we install python modules

docs/markdown/Commands.md

@@ -311,5 +311,14 @@ These variables are set in environment in addition to those set using `meson.add
   schemas is compiled. This is automatically set when using `gnome.compile_schemas()`.
   Note that this requires GLib >= 2.64 when `gnome.compile_schemas()` is used in
   more than one directory.
+- `PYTHONPATH` *Since 0.62.0* includes every directory where a python module is being
+  installed using [`python.install_sources()`](Python-module.md#install_sources)
+  and [`python.extension_module()`](Python-module.md#extension_module). Python
+  modules installed by other means, such as `install_data()` or `install_subdir()`,
+  will not be included and should be added to `PYTHONPATH` manually using
+  [`meson.add_devenv()`](Reference-manual_builtin_meson.md#mesonadd_devenv).
+  Note that when modules are installed into subdirectories the source tree
+  layout must match the installed tree layout otherwise `import subdir.mod`
+  cannot work.
 
 {{ devenv_arguments.inc }}

docs/markdown/snippets/devenv.md

@@ -0,0 +1,5 @@
+## `PYTHONPATH` automatically defined in `meson devenv`
+
+`PYTHONPATH` now includes every directory where a python module is being
+installed using [`python.install_sources()`](Python-module.md#install_sources)
+and [`python.extension_module()`](Python-module.md#extension_module).

mesonbuild/modules/python.py

@@ -23,7 +23,7 @@ from . import ExtensionModule
 from .. import mesonlib
 from .. import mlog
 from ..coredata import UserFeatureOption
-from ..build import known_shmod_kwargs
+from ..build import known_shmod_kwargs, EnvironmentVariables
 from ..dependencies import DependencyMethods, PkgConfigDependency, NotFoundDependency, SystemDependency, ExtraFrameworkDependency
 from ..dependencies.base import process_method_kw
 from ..environment import detect_cpu_family
@@ -45,6 +45,7 @@ if T.TYPE_CHECKING:
     from ..environment import Environment
     from ..interpreter import Interpreter
     from ..interpreterbase.interpreterbase import TYPE_var, TYPE_kwargs
+    from ..backends import InstallData
 
     from typing_extensions import TypedDict
 
@@ -389,6 +390,21 @@ class PythonExternalProgram(ExternalProgram):
             'variables': {},
             'version': '0.0',
         }
+        self.devenv_pythonpath: T.Set[str] = set()
+
+    def add_devenv_pythonpath(self, basedir: str, subdir: str, install_subdir: str) -> None:
+        # If we install python module into 'foo/bar' subdir, we need the last 2
+        # parts of source dir to be ['foo', 'bar'] and set PYTHONPATH
+        # pointing grandparent directory. That way scripts will be able to
+        # `import foo.bar.something` just like when the are installed.
+        # If the source tree layout does not match installed layout there is
+        # nothing we can do.
+        install_subdir_parts = Path(install_subdir).parts
+        subdir_parts = Path(subdir).parts
+        if subdir_parts[-len(install_subdir_parts):] == install_subdir_parts:
+            pypath = os.path.join(basedir, *subdir_parts[:-len(install_subdir_parts)])
+            self.devenv_pythonpath.add(pypath)
+            print('done', pypath)
 
     def _check_version(self, version: str) -> bool:
         if self.name == 'python2':
@@ -505,8 +521,10 @@ class PythonInstallation(ExternalProgramHolder):
             subdir = kwargs.pop('subdir', '')
             if not isinstance(subdir, str):
                 raise InvalidArguments('"subdir" argument must be a string.')
-
             kwargs['install_dir'] = os.path.join(self.platlib_install_path, subdir)
+            self.held_object.add_devenv_pythonpath(
+                self.interpreter.environment.get_build_dir(),
+                self.interpreter.subdir, subdir)
 
         # On macOS and some Linux distros (Debian) distutils doesn't link
         # extensions against libpython. We call into distutils and mirror its
@@ -561,11 +579,19 @@ class PythonInstallation(ExternalProgramHolder):
     def install_sources_method(self, args: T.Tuple[T.List[T.Union[str, mesonlib.File]]],
                                kwargs: 'PyInstallKw') -> 'Data':
         tag = kwargs['install_tag'] or 'runtime'
-        return self.interpreter.install_data_impl(
-            self.interpreter.source_strings_to_files(args[0]),
-            self._get_install_dir_impl(kwargs['pure'], kwargs['subdir']),
+        pure = kwargs['pure']
+        sources = self.interpreter.source_strings_to_files(args[0])
+        install_subdir = kwargs['subdir']
+        install_dir = self._get_install_dir_impl(pure, install_subdir)
+        builddir = self.interpreter.environment.get_build_dir()
+        srcdir = self.interpreter.environment.get_source_dir()
+        for src in sources:
+            basedir = builddir if src.is_built else srcdir
+            subdir = os.path.dirname(src.relative_name())
+            self.held_object.add_devenv_pythonpath(basedir, subdir, install_subdir)
+        return self.interpreter.install_data_impl(sources, install_dir,
             mesonlib.FileMode(), rename=None, tag=tag, install_data_type='python',
-            install_dir_name=self._get_install_dir_name_impl(kwargs['pure'], kwargs['subdir']))
+            install_dir_name=self._get_install_dir_name_impl(pure, install_subdir))
 
     @noPosargs
     @typed_kwargs('python_installation.install_dir', _PURE_KW, _SUBDIR_KW)
@@ -642,6 +668,18 @@ class PythonModule(ExtensionModule):
             'find_installation': self.find_installation,
         })
 
+    def get_devenv(self) -> T.Optional[EnvironmentVariables]:
+        pythonpath = set()
+        for python in self.installations.values():
+            version = python.info['version']
+            if mesonlib.version_compare(version, '>=3.0'):
+                pythonpath |= python.devenv_pythonpath
+        if pythonpath:
+            env = EnvironmentVariables()
+            env.prepend('PYTHONPATH', list(pythonpath))
+            return env
+        return None
+
     # https://www.python.org/dev/peps/pep-0397/
     @staticmethod
     def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]:

test cases/unit/91 devenv/meson.build

@@ -1,4 +1,8 @@
-project('devenv', 'c')
+project('devenv', 'c',
+  # Because Windows Python ships only with optimized libs,
+  # we must build this project the same way.
+  default_options : ['buildtype=release'],
+)
 
 meson.add_devenv('TEST_A=1')
 foo_dep = dependency('foo', fallback: 'sub')
@@ -10,3 +14,7 @@ meson.add_devenv(env)
 # This exe links on a library built in another directory. On Windows this means
 # PATH must contain builddir/subprojects/sub to be able to run it.
 executable('app', 'main.c', dependencies: foo_dep, install: true)
+
+py = import('python').find_installation()
+py.install_sources('src/mymod/mod.py', subdir: 'mymod')
+subdir('src/mymod2')

test cases/unit/91 devenv/src/mymod/mod.py

@@ -0,0 +1 @@
+hello = 'world'

test cases/unit/91 devenv/src/mymod2/meson.build

@@ -0,0 +1,5 @@
+py.extension_module('mod2', 'mod2.c',
+  dependencies: py.dependency(),
+  subdir: 'mymod2',
+  install: true
+)

test cases/unit/91 devenv/src/mymod2/mod2.c

@@ -0,0 +1,14 @@
+#include <Python.h>
+#include <string.h>
+
+static PyObject *hello(PyObject *self, PyObject *args) {
+  return PyLong_FromLong(42);
+}
+
+static PyMethodDef methods[] = {{"hello", hello, METH_NOARGS, "Hello World"},
+                                {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef mod = {PyModuleDef_HEAD_INIT, "test", NULL, -1,
+                                 methods};
+
+PyMODINIT_FUNC PyInit_mod2(void) { return PyModule_Create(&mod); }

test cases/unit/91 devenv/test-devenv.py

@@ -1,8 +1,15 @@
 #! /usr/bin/python
 
 import os
+from pathlib import Path
 
 assert os.environ['MESON_DEVENV'] == '1'
 assert os.environ['MESON_PROJECT_NAME'] == 'devenv'
 assert os.environ['TEST_A'] == '1'
 assert os.environ['TEST_B'] == '1+2+3'
+
+from mymod.mod import hello
+assert hello == 'world'
+
+from mymod2.mod2 import hello
+assert hello() == 42