Simplify module API

- ModuleState is now a real class that will have methods in the future
  for actions modules needs, instead of using interpreter internal API.
- New ModuleObject base class, similar to InterpreterObject, that should
  be used by all objects returned by modules. Its methods gets the
  ModuleState passed as first argument. It has a `methods` dictionary to
  define what is public API that can be called from build definition.
- Method return value is not required to be a ModuleReturnValue any
  more, it can be any type that interpreter can holderify, including
  ModuleObject.
- Legacy module API is maintained until we port all modules.

In the future modules should be updated:
- Use methods dict.
- Remove snippets.
- Custom objects returned by modules should all be subclass of
  ModuleObject to get the state iface in their methods.
- Modules should never call into interpreter directly and instead state
  object should have wrapper API.
- Stop using ModuleReturnValue in methods that just return simple
  objects like strings. Possibly remove ModuleReturnValue completely
  since all objects that needs to be processed by interpreter (e.g.
  CustomTarget) should be created through ModuleState API.

mesonbuild/interpreter.py

@@ -33,7 +33,7 @@ from .interpreterbase import FeatureNew, FeatureDeprecated, FeatureNewKwargs, Fe
 from .interpreterbase import ObjectHolder, MesonVersionString
 from .interpreterbase import TYPE_var, TYPE_nkwargs
 from .interpreterbase import typed_pos_args
-from .modules import ModuleReturnValue, ExtensionModule
+from .modules import ModuleReturnValue, ModuleObject, ModuleState
 from .cmake import CMakeInterpreter
 from .backend.backends import TestProtocol, Backend, ExecutableSerialisation
 
@@ -55,7 +55,6 @@ if T.TYPE_CHECKING:
     from .compilers import Compiler
     from .envconfig import MachineInfo
     from .environment import Environment
-    from .modules import ExtensionModule
 
 permitted_method_kwargs = {
     'partial_dependency': {'compile_args', 'link_args', 'links', 'includes',
@@ -1765,93 +1764,43 @@ class CompilerHolder(InterpreterObject):
         return self.compiler.get_argument_syntax()
 
 
-class ModuleState(T.NamedTuple):
+class ModuleObjectHolder(InterpreterObject, ObjectHolder['ModuleObject']):
-
+    def __init__(self, modobj: 'ModuleObject', interpreter: 'Interpreter'):
-    """Object passed to a module when it a method is called.
-
-    holds the current state of the meson process at a given method call in
-    the interpreter.
-    """
-
-    source_root: str
-    build_to_src: str
-    subproject: str
-    subdir: str
-    current_lineno: str
-    environment: 'Environment'
-    project_name: str
-    project_version: str
-    backend: str
-    targets: T.Dict[str, build.Target]
-    data: T.List[build.Data]
-    headers: T.List[build.Headers]
-    man: T.List[build.Man]
-    global_args: T.Dict[str, T.List[str]]
-    project_args: T.Dict[str, T.List[str]]
-    build_machine: 'MachineInfo'
-    host_machine: 'MachineInfo'
-    target_machine: 'MachineInfo'
-    current_node: mparser.BaseNode
-
-
-class ModuleHolder(InterpreterObject, ObjectHolder['ExtensionModule']):
-    def __init__(self, modname: str, module: 'ExtensionModule', interpreter: 'Interpreter'):
         InterpreterObject.__init__(self)
-        ObjectHolder.__init__(self, module)
+        ObjectHolder.__init__(self, modobj)
-        self.modname = modname
         self.interpreter = interpreter
 
     def method_call(self, method_name, args, kwargs):
-        try:
+        modobj = self.held_object
-            fn = getattr(self.held_object, method_name)
+        method = modobj.methods.get(method_name)
-        except AttributeError:
+        if not method and not modobj.methods:
-            raise InvalidArguments('Module %s does not have method %s.' % (self.modname, method_name))
+            # FIXME: Port all modules to use the methods dict.
-        if method_name.startswith('_'):
+            method = getattr(modobj, method_name, None)
-            raise InvalidArguments('Function {!r} in module {!r} is private.'.format(method_name, self.modname))
+            if method_name.startswith('_'):
-        if not getattr(fn, 'no-args-flattening', False):
+                raise InvalidArguments('Method {!r} is private.'.format(method_name))
+        if not method:
+            raise InvalidCode('Unknown method "%s" in object.' % method_name)
+        if not getattr(method, 'no-args-flattening', False):
             args = flatten(args)
-        # This is not 100% reliable but we can't use hash()
+        state = ModuleState(self.interpreter)
-        # because the Build object contains dicts and lists.
-        num_targets = len(self.interpreter.build.targets)
-        state = ModuleState(
-            source_root = self.interpreter.environment.get_source_dir(),
-            build_to_src=mesonlib.relpath(self.interpreter.environment.get_source_dir(),
-                                          self.interpreter.environment.get_build_dir()),
-            subproject=self.interpreter.subproject,
-            subdir=self.interpreter.subdir,
-            current_lineno=self.interpreter.current_lineno,
-            environment=self.interpreter.environment,
-            project_name=self.interpreter.build.project_name,
-            project_version=self.interpreter.build.dep_manifest[self.interpreter.active_projectname],
-            # The backend object is under-used right now, but we will need it:
-            # https://github.com/mesonbuild/meson/issues/1419
-            backend=self.interpreter.backend,
-            targets=self.interpreter.build.targets,
-            data=self.interpreter.build.data,
-            headers=self.interpreter.build.get_headers(),
-            man=self.interpreter.build.get_man(),
-            #global_args_for_build = self.interpreter.build.global_args.build,
-            global_args = self.interpreter.build.global_args.host,
-            #project_args_for_build = self.interpreter.build.projects_args.build.get(self.interpreter.subproject, {}),
-            project_args = self.interpreter.build.projects_args.host.get(self.interpreter.subproject, {}),
-            build_machine=self.interpreter.builtin['build_machine'].held_object,
-            host_machine=self.interpreter.builtin['host_machine'].held_object,
-            target_machine=self.interpreter.builtin['target_machine'].held_object,
-            current_node=self.current_node
-        )
         # Many modules do for example self.interpreter.find_program_impl(),
         # so we have to ensure they use the current interpreter and not the one
         # that first imported that module, otherwise it will use outdated
         # overrides.
-        self.held_object.interpreter = self.interpreter
+        modobj.interpreter = self.interpreter
-        if self.held_object.is_snippet(method_name):
+        if method_name in modobj.snippets:
-            value = fn(self.interpreter, state, args, kwargs)
+            ret = method(self.interpreter, state, args, kwargs)
-            return self.interpreter.holderify(value)
         else:
-            value = fn(state, args, kwargs)
+            # This is not 100% reliable but we can't use hash()
+            # because the Build object contains dicts and lists.
+            num_targets = len(self.interpreter.build.targets)
+            ret = method(state, args, kwargs)
             if num_targets != len(self.interpreter.build.targets):
                 raise InterpreterException('Extension module altered internal state illegally.')
-            return self.interpreter.module_method_callback(value)
+            if isinstance(ret, ModuleReturnValue):
+                self.interpreter.process_new_values(ret.new_objects)
+                ret = ret.return_value
+        return self.interpreter.holderify(ret)
 
 
 class Summary:
@@ -2401,7 +2350,7 @@ class Interpreter(InterpreterBase):
                 subproject: str = '',
                 subdir: str = '',
                 subproject_dir: str = 'subprojects',
-                modules: T.Optional[T.Dict[str, ExtensionModule]] = None,
+                modules: T.Optional[T.Dict[str, ModuleObject]] = None,
                 default_project_options: T.Optional[T.Dict[str, str]] = None,
                 mock: bool = False,
                 ast: T.Optional[mparser.CodeBlockNode] = None,
@@ -2566,6 +2515,8 @@ class Interpreter(InterpreterBase):
             return DependencyHolder(item, self.subproject)
         elif isinstance(item, dependencies.ExternalProgram):
             return ExternalProgramHolder(item, self.subproject)
+        elif isinstance(item, ModuleObject):
+            return ModuleObjectHolder(item, self)
         elif isinstance(item, (InterpreterObject, ObjectHolder)):
             return item
         else:
@@ -2600,13 +2551,6 @@ class Interpreter(InterpreterBase):
             else:
                 raise InterpreterException('Module returned a value of unknown type.')
 
-    def module_method_callback(self, return_object):
-        if not isinstance(return_object, ModuleReturnValue):
-            raise InterpreterException('Bug in module, it returned an invalid object')
-        invalues = return_object.new_objects
-        self.process_new_values(invalues)
-        return self.holderify(return_object.return_value)
-
     def get_build_def_files(self) -> T.List[str]:
         return self.build_def_files
 
@@ -2676,7 +2620,7 @@ class Interpreter(InterpreterBase):
         except ImportError:
             raise InvalidArguments('Module "%s" does not exist' % (modname, ))
         ext_module = module.initialize(self)
-        assert isinstance(ext_module, ExtensionModule)
+        assert isinstance(ext_module, ModuleObject)
         self.modules[modname] = ext_module
 
     @stringArgs
@@ -2696,7 +2640,7 @@ class Interpreter(InterpreterBase):
                 mlog.warning('Module %s has no backwards or forwards compatibility and might not exist in future releases.' % modname, location=node)
                 modname = 'unstable_' + plainname
         self.import_module(modname)
-        return ModuleHolder(modname, self.modules[modname], self)
+        return ModuleObjectHolder(self.modules[modname], self)
 
     @stringArgs
     @noKwargs

mesonbuild/modules/__init__.py

@@ -18,21 +18,58 @@
 import os
 
 from .. import build
-from ..mesonlib import unholder
+from ..mesonlib import unholder, relpath
 import typing as T
 
 if T.TYPE_CHECKING:
     from ..interpreter import Interpreter
-    from ..interpreterbase import TYPE_var
+    from ..interpreterbase import TYPE_var, TYPE_nvar, TYPE_nkwargs
+
+class ModuleState:
+    """Object passed to all module methods.
+
+    This is a WIP API provided to modules, it should be extended to have everything
+    needed so modules does not touch any other part of Meson internal APIs.
+    """
 
-class ExtensionModule:
     def __init__(self, interpreter: 'Interpreter') -> None:
+        self.source_root = interpreter.environment.get_source_dir()
+        self.build_to_src = relpath(interpreter.environment.get_source_dir(),
+                                    interpreter.environment.get_build_dir())
+        self.subproject = interpreter.subproject
+        self.subdir = interpreter.subdir
+        self.current_lineno = interpreter.current_lineno
+        self.environment = interpreter.environment
+        self.project_name = interpreter.build.project_name
+        self.project_version = interpreter.build.dep_manifest[interpreter.active_projectname]
+        # The backend object is under-used right now, but we will need it:
+        # https://github.com/mesonbuild/meson/issues/1419
+        self.backend = interpreter.backend
+        self.targets = interpreter.build.targets
+        self.data = interpreter.build.data
+        self.headers = interpreter.build.get_headers()
+        self.man = interpreter.build.get_man()
+        self.global_args = interpreter.build.global_args.host
+        self.project_args = interpreter.build.projects_args.host.get(interpreter.subproject, {})
+        self.build_machine = interpreter.builtin['build_machine'].held_object
+        self.host_machine = interpreter.builtin['host_machine'].held_object
+        self.target_machine = interpreter.builtin['target_machine'].held_object
+        self.current_node = interpreter.current_node
+
+class ModuleObject:
+    """Base class for all objects returned by modules
+    """
+    def __init__(self, interpreter: T.Optional['Interpreter'] = None) -> None:
+        self.methods = {}  # type: T.Dict[str, T.Callable[[T.List[TYPE_nvar], TYPE_nkwargs], TYPE_var]]
+        # FIXME: Port all modules to stop using self.interpreter and use API on
+        # ModuleState instead.
         self.interpreter = interpreter
-        self.snippets = set()  # type: T.Set[str] # List of methods that operate only on the interpreter.
+        # FIXME: Port all modules to remove snippets methods.
-
+        self.snippets: T.Set[str] = set()
-    def is_snippet(self, funcname: str) -> bool:
-        return funcname in self.snippets
 
+# FIXME: Port all modules to use ModuleObject directly.
+class ExtensionModule(ModuleObject):
+    pass
 
 def get_include_args(include_dirs, prefix='-I'):
     '''

mesonbuild/modules/fs.py

@@ -29,7 +29,8 @@ from ..mesonlib import (
 from ..interpreterbase import FeatureNew, typed_pos_args, noKwargs, permittedKwargs
 
 if T.TYPE_CHECKING:
-    from ..interpreter import Interpreter, ModuleState
+    from . import ModuleState
+    from ..interpreter import Interpreter
 
 
 class FSModule(ExtensionModule):

mesonbuild/modules/python.py

@@ -21,7 +21,6 @@ from pathlib import Path
 from .. import mesonlib
 from ..mesonlib import MachineChoice, MesonException
 from . import ExtensionModule
-from mesonbuild.modules import ModuleReturnValue
 from ..interpreterbase import (
     noPosargs, noKwargs, permittedKwargs,
     InvalidArguments,
@@ -399,12 +398,12 @@ class PythonInstallation(ExternalProgramHolder):
         else:
             res = os.path.join(self.platlib_install_path, subdir)
 
-        return self.interpreter.module_method_callback(ModuleReturnValue(res, []))
+        return res
 
     @noPosargs
     @noKwargs
     def language_version_method(self, args, kwargs):
-        return self.interpreter.module_method_callback(ModuleReturnValue(self.version, []))
+        return self.version
 
     @noKwargs
     def has_path_method(self, args, kwargs):
@@ -414,7 +413,7 @@ class PythonInstallation(ExternalProgramHolder):
         if not isinstance(path_name, str):
             raise InvalidArguments('has_path argument must be a string.')
 
-        return self.interpreter.module_method_callback(ModuleReturnValue(path_name in self.paths, []))
+        return path_name in self.paths
 
     @noKwargs
     def get_path_method(self, args, kwargs):
@@ -432,7 +431,7 @@ class PythonInstallation(ExternalProgramHolder):
             else:
                 raise InvalidArguments('{} is not a valid path name'.format(path_name))
 
-        return self.interpreter.module_method_callback(ModuleReturnValue(path, []))
+        return path
 
     @noKwargs
     def has_variable_method(self, args, kwargs):
@@ -442,7 +441,7 @@ class PythonInstallation(ExternalProgramHolder):
         if not isinstance(var_name, str):
             raise InvalidArguments('has_variable argument must be a string.')
 
-        return self.interpreter.module_method_callback(ModuleReturnValue(var_name in self.variables, []))
+        return var_name in self.variables
 
     @noKwargs
     def get_variable_method(self, args, kwargs):
@@ -460,7 +459,7 @@ class PythonInstallation(ExternalProgramHolder):
             else:
                 raise InvalidArguments('{} is not a valid variable name'.format(var_name))
 
-        return self.interpreter.module_method_callback(ModuleReturnValue(var, []))
+        return var
 
     @noPosargs
     @noKwargs

mesonbuild/modules/unstable_rust.py

@@ -24,7 +24,8 @@ from ..interpreterbase import InterpreterException, permittedKwargs, FeatureNew,
 from ..mesonlib import stringlistify, unholder, listify, typeslistify, File
 
 if T.TYPE_CHECKING:
-    from ..interpreter import ModuleState, Interpreter
+    from . import ModuleState
+    from ..interpreter import Interpreter
     from ..dependencies import ExternalProgram