dependencies: defer importing a custom dependency until it is used

This lessens the amount of code imported at Meson startup by mapping
each dependency to a dictionary entry and using a programmable import to
dynamically return it.

Minus 16 files and 6399 lines of code imported at startup.

mesonbuild/dependencies/__init__.py

@@ -12,35 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .boost import BoostDependency
-from .cuda import CudaDependency
-from .hdf5 import hdf5_factory
 from .base import Dependency, InternalDependency, ExternalDependency, NotFoundDependency, MissingCompiler
 from .base import (
         ExternalLibrary, DependencyException, DependencyMethods,
         BuiltinDependency, SystemDependency, get_leaf_external_dependencies)
-from .cmake import CMakeDependency
-from .configtool import ConfigToolDependency
-from .framework import ExtraFrameworkDependency
-from .pkgconfig import PkgConfigDependency
-from .factory import DependencyFactory
 from .detect import find_external_dependency, get_dep_identifier, packages, _packages_accept_language
-from .dev import (
-    ValgrindDependency, JNISystemDependency, JDKSystemDependency, gmock_factory, gtest_factory,
-    llvm_factory, zlib_factory)
-from .coarrays import coarray_factory
-from .mpi import mpi_factory
-from .scalapack import scalapack_factory
-from .misc import (
-    BlocksDependency, OpenMPDependency, cups_factory, curses_factory, gpgme_factory,
-    libgcrypt_factory, libwmf_factory, netcdf_factory, pcap_factory,
-    shaderc_factory, threads_factory, ThreadDependency, iconv_factory, intl_factory,
-    dl_factory, openssl_factory, libcrypto_factory, libssl_factory,
-)
-from .platform import AppleFrameworks
-from .python import python_factory as python3_factory, pybind11_factory
-from .qt import qt4_factory, qt5_factory, qt6_factory
-from .ui import GnuStepDependency, WxDependency, gl_factory, sdl2_factory, vulkan_factory
 
 __all__ = [
     'Dependency',
@@ -54,15 +30,6 @@ __all__ = [
     'DependencyMethods',
     'MissingCompiler',
 
-    'CMakeDependency',
-    'ConfigToolDependency',
-    'ExtraFrameworkDependency',
-    'PkgConfigDependency',
-
-    'DependencyFactory',
-
-    'ThreadDependency',
-
     'find_external_dependency',
     'get_dep_identifier',
     'get_leaf_external_dependencies',
@@ -225,60 +192,62 @@ this approach, and no new dependencies should do this.
 # - An ExternalDependency subclass
 # - A DependencyFactory object
 # - A callable with a signature of (Environment, MachineChoice, Dict[str, Any]) -> List[Callable[[], ExternalDependency]]
-packages.update({
+packages.defaults.update({
     # From dev:
-    'gtest': gtest_factory,
-    'gmock': gmock_factory,
-    'llvm': llvm_factory,
-    'valgrind': ValgrindDependency,
-    'zlib': zlib_factory,
-    'jni': JNISystemDependency,
-    'jdk': JDKSystemDependency,
+    'gtest': 'dev:gtest_factory',
+    'gmock': 'dev:gmock_factory',
+    'llvm': 'dev:llvm_factory',
+    'valgrind': 'dev:ValgrindDependency',
+    'zlib': 'dev:zlib_factory',
+    'jni': 'dev:JNISystemDependency',
+    'jdk': 'dev:JDKSystemDependency',
 
-    'boost': BoostDependency,
-    'cuda': CudaDependency,
+    'boost': 'boost:BoostDependency',
+    'cuda': 'cuda:CudaDependency',
 
     # per-file
-    'coarray': coarray_factory,
-    'hdf5': hdf5_factory,
-    'mpi': mpi_factory,
-    'scalapack': scalapack_factory,
+    'coarray': 'coarrays:coarray_factory',
+    'hdf5': 'hdf5:hdf5_factory',
+    'mpi': 'mpi:mpi_factory',
+    'scalapack': 'scalapack:scalapack_factory',
 
     # From misc:
-    'blocks': BlocksDependency,
-    'curses': curses_factory,
-    'netcdf': netcdf_factory,
-    'openmp': OpenMPDependency,
-    'threads': threads_factory,
-    'pcap': pcap_factory,
-    'cups': cups_factory,
-    'libwmf': libwmf_factory,
-    'libgcrypt': libgcrypt_factory,
-    'gpgme': gpgme_factory,
-    'shaderc': shaderc_factory,
-    'iconv': iconv_factory,
-    'intl': intl_factory,
-    'dl': dl_factory,
-    'openssl': openssl_factory,
-    'libcrypto': libcrypto_factory,
-    'libssl': libssl_factory,
+    'blocks': 'misc:BlocksDependency',
+    'curses': 'misc:curses_factory',
+    'netcdf': 'misc:netcdf_factory',
+    'openmp': 'misc:OpenMPDependency',
+    'threads': 'misc:threads_factory',
+    'pcap': 'misc:pcap_factory',
+    'cups': 'misc:cups_factory',
+    'libwmf': 'misc:libwmf_factory',
+    'libgcrypt': 'misc:libgcrypt_factory',
+    'gpgme': 'misc:gpgme_factory',
+    'shaderc': 'misc:shaderc_factory',
+    'iconv': 'misc:iconv_factory',
+    'intl': 'misc:intl_factory',
+    'dl': 'misc:dl_factory',
+    'openssl': 'misc:openssl_factory',
+    'libcrypto': 'misc:libcrypto_factory',
+    'libssl': 'misc:libssl_factory',
 
     # From platform:
-    'appleframeworks': AppleFrameworks,
+    'appleframeworks': 'platform:AppleFrameworks',
 
     # from python:
-    'python3': python3_factory,
-    'pybind11': pybind11_factory,
+    'python3': 'python:python_factory',
+    'pybind11': 'python:pybind11_factory',
 
     # From ui:
-    'gl': gl_factory,
-    'gnustep': GnuStepDependency,
-    'qt4': qt4_factory,
-    'qt5': qt5_factory,
-    'qt6': qt6_factory,
-    'sdl2': sdl2_factory,
-    'wxwidgets': WxDependency,
-    'vulkan': vulkan_factory,
+    'gl': 'ui:gl_factory',
+    'gnustep': 'ui:GnuStepDependency',
+    'sdl2': 'ui:sdl2_factory',
+    'wxwidgets': 'ui:WxDependency',
+    'vulkan': 'ui:vulkan_factory',
+
+    # from qt
+    'qt4': 'qt:qt4_factory',
+    'qt5': 'qt:qt5_factory',
+    'qt6': 'qt:qt6_factory',
 })
 _packages_accept_language.update({
     'hdf5',

mesonbuild/dependencies/detect.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 from __future__ import annotations
 
+import collections, functools, importlib
+import typing as T
+
 from .base import ExternalDependency, DependencyException, DependencyMethods, NotFoundDependency
 
 from ..mesonlib import listify, MachineChoice, PerMachine
 from .. import mlog
-import functools
-import typing as T
 
 if T.TYPE_CHECKING:
     from ..environment import Environment
@@ -26,12 +27,27 @@ if T.TYPE_CHECKING:
 
     TV_DepIDEntry = T.Union[str, bool, int, T.Tuple[str, ...]]
     TV_DepID = T.Tuple[T.Tuple[str, TV_DepIDEntry], ...]
+    PackageTypes = T.Union[T.Type[ExternalDependency], DependencyFactory, WrappedFactoryFunc]
+
+class DependencyPackages(collections.UserDict):
+    data: T.Dict[str, PackageTypes]
+    defaults: T.Dict[str, str] = {}
+
+    def __missing__(self, key: str) -> PackageTypes:
+        if key in self.defaults:
+            modn, package = self.defaults[key].split(':', maxsplit=1)
+            mod = importlib.import_module(f'mesonbuild.dependencies.{modn}')
+            value = T.cast('PackageTypes', getattr(mod, package))
+            self.data[key] = value
+
+            return value
+        raise KeyError(key)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self.defaults or key in self.data
 
 # These must be defined in this file to avoid cyclical references.
-packages: T.Dict[
-    str,
-    T.Union[T.Type[ExternalDependency], 'DependencyFactory', 'WrappedFactoryFunc']
-] = {}
+packages = DependencyPackages()
 _packages_accept_language: T.Set[str] = set()
 
 def get_dep_identifier(name: str, kwargs: T.Dict[str, T.Any]) -> 'TV_DepID':

mesonbuild/mdevenv.py

@@ -7,7 +7,7 @@ import shutil
 import itertools
 
 from pathlib import Path
-from . import build, minstall, dependencies
+from . import build, minstall
 from .mesonlib import (MesonException, is_windows, setup_vsenv, OptionKey,
                        get_wine_shortpath, MachineChoice)
 from . import mlog
@@ -75,9 +75,10 @@ def get_env(b: build.Build, dump_fmt: T.Optional[str]) -> T.Tuple[T.Dict[str, st
     return env, varnames
 
 def bash_completion_files(b: build.Build, install_data: 'InstallData') -> T.List[str]:
+    from .dependencies.pkgconfig import PkgConfigDependency
     result = []
-    dep = dependencies.PkgConfigDependency('bash-completion', b.environment,
-                                           {'required': False, 'silent': True, 'version': '>=2.10'})
+    dep = PkgConfigDependency('bash-completion', b.environment,
+                              {'required': False, 'silent': True, 'version': '>=2.10'})
     if dep.found():
         prefix = b.environment.coredata.get_option(OptionKey('prefix'))
         assert isinstance(prefix, str), 'for mypy'

mesonbuild/modules/external_project.py

@@ -23,7 +23,8 @@ from . import ExtensionModule, ModuleReturnValue, NewExtensionModule, ModuleInfo
 from .. import mlog, build
 from ..compilers.compilers import CFLAGS_MAPPING
 from ..envconfig import ENV_VAR_PROG_MAP
-from ..dependencies import InternalDependency, PkgConfigDependency
+from ..dependencies import InternalDependency
+from ..dependencies.pkgconfig import PkgConfigDependency
 from ..interpreterbase import FeatureNew
 from ..interpreter.type_checking import ENV_KW, DEPENDS_KW
 from ..interpreterbase.decorators import ContainerTypeInfo, KwargInfo, typed_kwargs, typed_pos_args

mesonbuild/modules/gnome.py

@@ -31,7 +31,8 @@ from .. import interpreter
 from .. import mesonlib
 from .. import mlog
 from ..build import CustomTarget, CustomTargetIndex, Executable, GeneratedList, InvalidArguments
-from ..dependencies import Dependency, PkgConfigDependency, InternalDependency
+from ..dependencies import Dependency, InternalDependency
+from ..dependencies.pkgconfig import PkgConfigDependency
 from ..interpreter.type_checking import DEPENDS_KW, DEPEND_FILES_KW, ENV_KW, INSTALL_DIR_KW, INSTALL_KW, NoneType, SOURCES_KW, in_set_validator
 from ..interpreterbase import noPosargs, noKwargs, FeatureNew, FeatureDeprecated
 from ..interpreterbase import typed_kwargs, KwargInfo, ContainerTypeInfo

run_project_tests.py

@@ -540,7 +540,7 @@ def validate_output(test: TestDef, stdo: str, stde: str) -> str:
 # coded to run as a batch process.
 def clear_internal_caches() -> None:
     import mesonbuild.interpreterbase
-    from mesonbuild.dependencies import CMakeDependency
+    from mesonbuild.dependencies.cmake import CMakeDependency
     from mesonbuild.mesonlib import PerMachine
     mesonbuild.interpreterbase.FeatureNew.feature_registry = {}
     CMakeDependency.class_cmakeinfo = PerMachine(None, None)

run_tests.py

@@ -36,7 +36,7 @@ import typing as T
 
 from mesonbuild.compilers.c import CCompiler
 from mesonbuild.compilers.detect import detect_c_compiler
-from mesonbuild import dependencies
+from mesonbuild.dependencies.pkgconfig import PkgConfigDependency
 from mesonbuild import mesonlib
 from mesonbuild import mesonmain
 from mesonbuild import mtest
@@ -302,8 +302,8 @@ def run_mtest_inprocess(commandlist: T.List[str]) -> T.Tuple[int, str, str]:
 def clear_meson_configure_class_caches() -> None:
     CCompiler.find_library_cache = {}
     CCompiler.find_framework_cache = {}
-    dependencies.PkgConfigDependency.pkgbin_cache = {}
-    dependencies.PkgConfigDependency.class_pkgbin = mesonlib.PerMachine(None, None)
+    PkgConfigDependency.pkgbin_cache = {}
+    PkgConfigDependency.class_pkgbin = mesonlib.PerMachine(None, None)
     mesonlib.project_meson_versions = collections.defaultdict(str)
 
 def run_configure_inprocess(commandlist: T.List[str], env: T.Optional[T.Dict[str, str]] = None, catch_exception: bool = False) -> T.Tuple[int, str, str]:

test cases/unit/113 empty project/expected_mods.json

@@ -201,24 +201,8 @@
       "mesonbuild.coredata",
       "mesonbuild.dependencies",
       "mesonbuild.dependencies.base",
-      "mesonbuild.dependencies.boost",
-      "mesonbuild.dependencies.cmake",
-      "mesonbuild.dependencies.coarrays",
-      "mesonbuild.dependencies.configtool",
-      "mesonbuild.dependencies.cuda",
       "mesonbuild.dependencies.detect",
-      "mesonbuild.dependencies.dev",
-      "mesonbuild.dependencies.factory",
-      "mesonbuild.dependencies.framework",
-      "mesonbuild.dependencies.hdf5",
-      "mesonbuild.dependencies.misc",
-      "mesonbuild.dependencies.mpi",
       "mesonbuild.dependencies.pkgconfig",
-      "mesonbuild.dependencies.platform",
-      "mesonbuild.dependencies.python",
-      "mesonbuild.dependencies.qt",
-      "mesonbuild.dependencies.scalapack",
-      "mesonbuild.dependencies.ui",
       "mesonbuild.depfile",
       "mesonbuild.envconfig",
       "mesonbuild.environment",
@@ -271,6 +255,6 @@
       "mesonbuild.wrap",
       "mesonbuild.wrap.wrap"
     ],
-    "count": 97
+    "count": 81
   }
 }

unittests/allplatformstests.py

@@ -56,7 +56,7 @@ from mesonbuild.compilers import (
     detect_compiler_for
 )
 
-from mesonbuild.dependencies import PkgConfigDependency
+from mesonbuild.dependencies.pkgconfig import PkgConfigDependency
 from mesonbuild.build import Target, ConfigurationData, Executable, SharedLibrary, StaticLibrary
 import mesonbuild.modules.pkgconfig
 from mesonbuild.scripts import destdir_join

unittests/internaltests.py

@@ -46,7 +46,7 @@ from mesonbuild.mesonlib import (
     OptionType
 )
 from mesonbuild.interpreter.type_checking import in_set_validator, NoneType
-from mesonbuild.dependencies import PkgConfigDependency
+from mesonbuild.dependencies.pkgconfig import PkgConfigDependency
 from mesonbuild.programs import ExternalProgram
 import mesonbuild.modules.pkgconfig
 

unittests/linuxliketests.py

@@ -45,7 +45,7 @@ from mesonbuild.compilers.c import AppleClangCCompiler
 from mesonbuild.compilers.cpp import AppleClangCPPCompiler
 from mesonbuild.compilers.objc import AppleClangObjCCompiler
 from mesonbuild.compilers.objcpp import AppleClangObjCPPCompiler
-from mesonbuild.dependencies import PkgConfigDependency
+from mesonbuild.dependencies.pkgconfig import PkgConfigDependency
 import mesonbuild.modules.pkgconfig
 
 PKG_CONFIG = os.environ.get('PKG_CONFIG', 'pkg-config')

unittests/platformagnostictests.py

@@ -253,4 +253,4 @@ class PlatformAgnosticTests(BasePlatformTests):
             expected = json.load(f)['meson']['modules']
 
         self.assertEqual(data['modules'], expected)
-        self.assertEqual(data['count'], 97)
+        self.assertEqual(data['count'], 81)