typing: Fully annotate dependencies.{detect,factory} + some other fixes

mesonbuild/coredata.py

@@ -21,7 +21,7 @@ from collections import OrderedDict
 from .mesonlib import (
     MesonException, EnvironmentException, MachineChoice, PerMachine,
     PerMachineDefaultable, default_libdir, default_libexecdir,
-    default_prefix, split_args, OptionKey, OptionType,
+    default_prefix, split_args, OptionKey, OptionType, stringlistify,
 )
 from .wrap import WrapMode
 import ast
@@ -266,8 +266,7 @@ class UserFeatureOption(UserComboOption):
         return self.value == 'auto'
 
 if T.TYPE_CHECKING:
-    CacheKeyType = T.Tuple[T.Tuple[T.Any, ...], ...]
-    SubCacheKeyType = T.Tuple[T.Any, ...]
+    from .dependencies.detect import TV_DepIDEntry, TV_DepID
 
 
 class DependencyCacheType(enum.Enum):
@@ -291,15 +290,15 @@ class DependencySubCache:
 
     def __init__(self, type_: DependencyCacheType):
         self.types = [type_]
-        self.__cache = {}  # type: T.Dict[SubCacheKeyType, dependencies.Dependency]
+        self.__cache: T.Dict[T.Tuple[str, ...], 'dependencies.Dependency'] = {}
 
-    def __getitem__(self, key: 'SubCacheKeyType') -> 'dependencies.Dependency':
+    def __getitem__(self, key: T.Tuple[str, ...]) -> 'dependencies.Dependency':
         return self.__cache[key]
 
-    def __setitem__(self, key: 'SubCacheKeyType', value: 'dependencies.Dependency') -> None:
+    def __setitem__(self, key: T.Tuple[str, ...], value: 'dependencies.Dependency') -> None:
         self.__cache[key] = value
 
-    def __contains__(self, key: 'SubCacheKeyType') -> bool:
+    def __contains__(self, key: T.Tuple[str, ...]) -> bool:
         return key in self.__cache
 
     def values(self) -> T.Iterable['dependencies.Dependency']:
@@ -315,30 +314,31 @@ class DependencyCache:
     """
 
     def __init__(self, builtins: 'KeyedOptionDictType', for_machine: MachineChoice):
-        self.__cache = OrderedDict()  # type: T.MutableMapping[CacheKeyType, DependencySubCache]
+        self.__cache = OrderedDict()  # type: T.MutableMapping[TV_DepID, DependencySubCache]
         self.__builtins = builtins
         self.__pkg_conf_key = OptionKey('pkg_config_path', machine=for_machine)
         self.__cmake_key = OptionKey('cmake_prefix_path', machine=for_machine)
 
-    def __calculate_subkey(self, type_: DependencyCacheType) -> T.Tuple[T.Any, ...]:
-        if type_ is DependencyCacheType.PKG_CONFIG:
-            return tuple(self.__builtins[self.__pkg_conf_key].value)
-        elif type_ is DependencyCacheType.CMAKE:
-            return tuple(self.__builtins[self.__cmake_key].value)
-        assert type_ is DependencyCacheType.OTHER, 'Someone forgot to update subkey calculations for a new type'
-        return tuple()
+    def __calculate_subkey(self, type_: DependencyCacheType) -> T.Tuple[str, ...]:
+        data: T.Dict[str, T.List[str]] = {
+            DependencyCacheType.PKG_CONFIG: stringlistify(self.__builtins[self.__pkg_conf_key].value),
+            DependencyCacheType.CMAKE: stringlistify(self.__builtins[self.__cmake_key].value),
+            DependencyCacheType.OTHER: [],
+        }
+        assert type_ in data, 'Someone forgot to update subkey calculations for a new type'
+        return tuple(data[type_])
 
-    def __iter__(self) -> T.Iterator['CacheKeyType']:
+    def __iter__(self) -> T.Iterator['TV_DepID']:
         return self.keys()
 
-    def put(self, key: 'CacheKeyType', dep: 'dependencies.Dependency') -> None:
+    def put(self, key: 'TV_DepID', dep: 'dependencies.Dependency') -> None:
         t = DependencyCacheType.from_type(dep)
         if key not in self.__cache:
             self.__cache[key] = DependencySubCache(t)
         subkey = self.__calculate_subkey(t)
         self.__cache[key][subkey] = dep
 
-    def get(self, key: 'CacheKeyType') -> T.Optional['dependencies.Dependency']:
+    def get(self, key: 'TV_DepID') -> T.Optional['dependencies.Dependency']:
         """Get a value from the cache.
 
         If there is no cache entry then None will be returned.
@@ -360,10 +360,10 @@ class DependencyCache:
         for c in self.__cache.values():
             yield from c.values()
 
-    def keys(self) -> T.Iterator['CacheKeyType']:
+    def keys(self) -> T.Iterator['TV_DepID']:
         return iter(self.__cache.keys())
 
-    def items(self) -> T.Iterator[T.Tuple['CacheKeyType', T.List['dependencies.Dependency']]]:
+    def items(self) -> T.Iterator[T.Tuple['TV_DepID', T.List['dependencies.Dependency']]]:
         for k, v in self.__cache.items():
             vs = []
             for t in v.types:

mesonbuild/dependencies/coarrays.py

@@ -22,14 +22,15 @@ from .factory import factory_methods
 
 if T.TYPE_CHECKING:
     from . base import Dependency
+    from . factory import TV_DepGenerators
     from ..environment import Environment, MachineChoice
 
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CMAKE, DependencyMethods.SYSTEM})
 def coarray_factory(env: 'Environment', for_machine: 'MachineChoice',
-                    kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List[T.Callable[[], 'Dependency']]:
+                    kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     fcid = detect_compiler('coarray', env, for_machine, 'fortran').get_id()
-    candidates: T.List[T.Callable[[], 'Dependency']] = []
+    candidates: 'TV_DepGenerators' = []
 
     if fcid == 'gcc':
         # OpenCoarrays is the most commonly used method for Fortran Coarray with GCC

mesonbuild/dependencies/detect.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base import Dependency, DependencyException, DependencyMethods, NotFoundDependency
+from .base import Dependency, ExternalDependency, DependencyException, DependencyMethods, NotFoundDependency
 from .cmake import CMakeDependency
 from .dub import DubDependency
 from .framework import ExtraFrameworkDependency
@@ -25,15 +25,19 @@ import typing as T
 
 if T.TYPE_CHECKING:
     from ..environment import Environment
-    from .base import ExternalDependency
-    from .factory import DependencyType
+    from .factory import DependencyFactory, TV_FactoryFunc, TV_DepGenerators
 
 # These must be defined in this file to avoid cyclical references.
-packages = {}
-_packages_accept_language = set()
+packages: T.Dict[str, T.Union[T.Type[ExternalDependency], 'DependencyFactory', 'TV_FactoryFunc']] = {}
+_packages_accept_language: T.Set[str] = set()
 
-def get_dep_identifier(name, kwargs) -> T.Tuple:
-    identifier = (name, )
+if T.TYPE_CHECKING:
+    TV_DepIDEntry = T.Union[str, bool, int, T.Tuple[str, ...]]
+    TV_DepID = T.Tuple[T.Tuple[str, TV_DepIDEntry], ...]
+
+
+def get_dep_identifier(name: str, kwargs: T.Dict[str, T.Any]) -> 'TV_DepID':
+    identifier: 'TV_DepID' = (('name', name), )
     from ..interpreter import permitted_dependency_kwargs
     assert len(permitted_dependency_kwargs) == 19, \
            'Extra kwargs have been added to dependency(), please review if it makes sense to handle it here'
@@ -53,6 +57,10 @@ def get_dep_identifier(name, kwargs) -> T.Tuple:
         # All keyword arguments are strings, ints, or lists (or lists of lists)
         if isinstance(value, list):
             value = frozenset(listify(value))
+            for i in value:
+                assert isinstance(i, str)
+        else:
+            assert isinstance(value, (str, bool, int))
         identifier += (key, value)
     return identifier
 
@@ -70,7 +78,7 @@ display_name_map = {
     'wxwidgets': 'WxWidgets',
 }
 
-def find_external_dependency(name: str, env: 'Environment', kwargs: T.Dict[str, object]) -> 'ExternalDependency':
+def find_external_dependency(name: str, env: 'Environment', kwargs: T.Dict[str, object]) -> T.Union['ExternalDependency', NotFoundDependency]:
     assert(name)
     required = kwargs.get('required', True)
     if not isinstance(required, bool):
@@ -94,7 +102,7 @@ def find_external_dependency(name: str, env: 'Environment', kwargs: T.Dict[str,
     candidates = _build_external_dependency_list(name, env, for_machine, kwargs)
 
     pkg_exc: T.List[DependencyException] = []
-    pkgdep: T.List['ExternalDependency'] = []
+    pkgdep:  T.List[ExternalDependency]  = []
     details = ''
 
     for c in candidates:
@@ -117,7 +125,7 @@ def find_external_dependency(name: str, env: 'Environment', kwargs: T.Dict[str,
             # if the dependency was found
             if d.found():
 
-                info = []
+                info: mlog.TV_LoggableList = []
                 if d.version:
                     info.append(mlog.normal_cyan(d.version))
 
@@ -155,7 +163,7 @@ def find_external_dependency(name: str, env: 'Environment', kwargs: T.Dict[str,
 
 
 def _build_external_dependency_list(name: str, env: 'Environment', for_machine: MachineChoice,
-                                    kwargs: T.Dict[str, T.Any]) -> T.List[T.Callable[[], 'ExternalDependency']]:
+                                    kwargs: T.Dict[str, T.Any]) -> 'TV_DepGenerators':
     # First check if the method is valid
     if 'method' in kwargs and kwargs['method'] not in [e.value for e in DependencyMethods]:
         raise DependencyException('method {!r} is invalid'.format(kwargs['method']))
@@ -166,13 +174,18 @@ def _build_external_dependency_list(name: str, env: 'Environment', for_machine:
         # Create the list of dependency object constructors using a factory
         # class method, if one exists, otherwise the list just consists of the
         # constructor
-        if isinstance(packages[lname], type) and issubclass(packages[lname], Dependency):
-            dep = [functools.partial(packages[lname], env, kwargs)]
+        if isinstance(packages[lname], type):
+            entry1 = T.cast(T.Type[ExternalDependency], packages[lname])  # mypy doesn't understand isinstance(..., type)
+            if issubclass(entry1, ExternalDependency):
+                # TODO: somehow make mypy understand that entry1(env, kwargs) is OK...
+                func: T.Callable[[], 'ExternalDependency'] = lambda: entry1(env, kwargs)  # type: ignore
+                dep = [func]
         else:
-            dep = packages[lname](env, for_machine, kwargs)
+            entry2 = T.cast(T.Union['DependencyFactory', 'TV_FactoryFunc'], packages[lname])
+            dep = entry2(env, for_machine, kwargs)
         return dep
 
-    candidates = []
+    candidates: 'TV_DepGenerators' = []
 
     # If it's explicitly requested, use the dub detection method (only)
     if 'dub' == kwargs.get('method', ''):

mesonbuild/dependencies/factory.py

@@ -24,9 +24,27 @@ import typing as T
 
 if T.TYPE_CHECKING:
     from ..environment import Environment
-    from .base import DependencyType
     from .configtool import ConfigToolDependency
-    FactoryType = T.TypeVar('FactoryType', bound=T.Callable[..., T.List[T.Callable[[], 'Dependency']]])
+
+    TV_DepGenerators = T.List[T.Callable[[], ExternalDependency]]
+    TV_WrappedFactoryFunc = T.Callable[
+        [
+            'Environment',
+            MachineChoice,
+            T.Dict[str, T.Any],
+            T.List[DependencyMethods]
+        ],
+        TV_DepGenerators
+    ]
+
+    TV_FactoryFunc = T.Callable[
+        [
+            'Environment',
+            MachineChoice,
+            T.Dict[str, T.Any]
+        ],
+        TV_DepGenerators
+    ]
 
 class DependencyFactory:
 
@@ -67,17 +85,20 @@ class DependencyFactory:
 
         self.extra_kwargs = extra_kwargs or {}
         self.methods = methods
-        self.classes = {
+        self.classes: T.Dict[
+            DependencyMethods,
+            T.Callable[['Environment', T.Dict[str, T.Any]], ExternalDependency]
+        ] = {
             # Just attach the correct name right now, either the generic name
             # or the method specific name.
-            DependencyMethods.EXTRAFRAMEWORK: functools.partial(framework_class, framework_name or name),
-            DependencyMethods.PKGCONFIG: functools.partial(pkgconfig_class, pkgconfig_name or name),
-            DependencyMethods.CMAKE: functools.partial(cmake_class, cmake_name or name),
-            DependencyMethods.SYSTEM: functools.partial(system_class, name),
-            DependencyMethods.CONFIG_TOOL: None,
+            DependencyMethods.EXTRAFRAMEWORK: lambda env, kwargs: framework_class(framework_name or name, env, kwargs),
+            DependencyMethods.PKGCONFIG:      lambda env, kwargs: pkgconfig_class(pkgconfig_name or name, env, kwargs),
+            DependencyMethods.CMAKE:          lambda env, kwargs: cmake_class(cmake_name or name, env, kwargs),
+            DependencyMethods.SYSTEM:         lambda env, kwargs: system_class(name, env, kwargs),
+            DependencyMethods.CONFIG_TOOL:    None,
         }
         if configtool_class is not None:
-            self.classes[DependencyMethods.CONFIG_TOOL] = functools.partial(configtool_class, name)
+            self.classes[DependencyMethods.CONFIG_TOOL] = lambda env, kwargs: configtool_class(name, env, kwargs)
 
     @staticmethod
     def _process_method(method: DependencyMethods, env: 'Environment', for_machine: MachineChoice) -> bool:
@@ -95,7 +116,7 @@ class DependencyFactory:
         return True
 
     def __call__(self, env: 'Environment', for_machine: MachineChoice,
-                 kwargs: T.Dict[str, T.Any]) -> T.List['DependencyType']:
+                 kwargs: T.Dict[str, T.Any]) -> 'TV_DepGenerators':
         """Return a list of Dependencies with the arguments already attached."""
         methods = process_method_kw(self.methods, kwargs)
         nwargs = self.extra_kwargs.copy()
@@ -105,21 +126,21 @@ class DependencyFactory:
                 if self._process_method(m, env, for_machine)]
 
 
-def factory_methods(methods: T.Set[DependencyMethods]) -> T.Callable[['FactoryType'], 'FactoryType']:
+def factory_methods(methods: T.Set[DependencyMethods]) -> T.Callable[['TV_WrappedFactoryFunc'], 'TV_FactoryFunc']:
     """Decorator for handling methods for dependency factory functions.
 
     This helps to make factory functions self documenting
     >>> @factory_methods([DependencyMethods.PKGCONFIG, DependencyMethods.CMAKE])
-    >>> def factory(env: Environment, for_machine: MachineChoice, kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List[T.Callable[[], 'Dependency']]:
+    >>> def factory(env: Environment, for_machine: MachineChoice, kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     >>>     pass
     """
 
-    def inner(func: 'FactoryType') -> 'FactoryType':
+    def inner(func: 'TV_WrappedFactoryFunc') -> 'TV_FactoryFunc':
 
         @functools.wraps(func)
-        def wrapped(env: 'Environment', for_machine: MachineChoice, kwargs: T.Dict[str, T.Any]) -> T.List[T.Callable[[], 'Dependency']]:
+        def wrapped(env: 'Environment', for_machine: MachineChoice, kwargs: T.Dict[str, T.Any]) -> 'TV_DepGenerators':
             return func(env, for_machine, kwargs, process_method_kw(methods, kwargs))
 
-        return T.cast('FactoryType', wrapped)
+        return wrapped
 
     return inner

mesonbuild/dependencies/hdf5.py

@@ -30,6 +30,7 @@ import typing as T
 
 if T.TYPE_CHECKING:
     from .base import Dependency
+    from .factory import TV_DepGenerators
     from ..environment import Environment
     from ..mesonlib import MachineChoice
 
@@ -143,7 +144,7 @@ class HDF5ConfigToolDependency(ConfigToolDependency):
             nkwargs = kwargs.copy()
             nkwargs['language'] = 'c'
             # I'm being too clever for mypy and pylint
-            self.is_found = self._add_sub_dependency(hdf5_factory(environment, for_machine, nkwargs))  # type: ignore  # pylint: disable=no-value-for-parameter
+            self.is_found = self._add_sub_dependency(hdf5_factory(environment, for_machine, nkwargs))  # pylint: disable=no-value-for-parameter
 
     def _sanitize_version(self, ver: str) -> str:
         v = re.search(r'\s*HDF5 Version: (\d+\.\d+\.\d+)', ver)
@@ -152,9 +153,9 @@ class HDF5ConfigToolDependency(ConfigToolDependency):
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL})
 def hdf5_factory(env: 'Environment', for_machine: 'MachineChoice',
-                 kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List[T.Callable[[], 'Dependency']]:
+                 kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     language = kwargs.get('language')
-    candidates = []  # type: T.List[T.Callable[[], Dependency]]
+    candidates: 'TV_DepGenerators' = []
 
     if DependencyMethods.PKGCONFIG in methods:
         # Use an ordered set so that these remain the first tried pkg-config files

mesonbuild/dependencies/misc.py

@@ -32,12 +32,13 @@ from .factory import DependencyFactory, factory_methods
 
 if T.TYPE_CHECKING:
     from ..environment import Environment, MachineChoice
+    from .factory import TV_DepGenerators
     from .base import DependencyType, Dependency  # noqa: F401
 
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CMAKE})
 def netcdf_factory(env: 'Environment', for_machine: 'MachineChoice',
-                   kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List['DependencyType']:
+                   kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     language = kwargs.get('language', 'c')
     if language not in ('c', 'cpp', 'fortran'):
         raise DependencyException(f'Language {language} is not supported with NetCDF.')
@@ -487,8 +488,8 @@ class CursesSystemDependency(ExternalDependency):
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.SYSTEM})
 def curses_factory(env: 'Environment', for_machine: 'MachineChoice',
-                   kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List[T.Callable[[], 'Dependency']]:
-    candidates = []  # type: T.List[T.Callable[[], Dependency]]
+                   kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
+    candidates: 'TV_DepGenerators' = []
 
     if DependencyMethods.PKGCONFIG in methods:
         pkgconfig_files = ['pdcurses', 'ncursesw', 'ncurses', 'curses']
@@ -510,7 +511,7 @@ def curses_factory(env: 'Environment', for_machine: 'MachineChoice',
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM})
 def shaderc_factory(env: 'Environment', for_machine: 'MachineChoice',
-                    kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List['DependencyType']:
+                    kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     """Custom DependencyFactory for ShaderC.
 
     ShaderC's odd you get three different libraries from the same build

mesonbuild/dependencies/mpi.py

@@ -25,6 +25,7 @@ from ..environment import detect_cpu_family
 
 if T.TYPE_CHECKING:
     from .base import Dependency
+    from .factory import TV_DepGenerators
     from ..compilers import Compiler
     from ..compilers.compiler import CompilerType
     from ..environment import Environment, MachineChoice
@@ -32,13 +33,13 @@ if T.TYPE_CHECKING:
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.SYSTEM})
 def mpi_factory(env: 'Environment', for_machine: 'MachineChoice',
-                kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List[T.Callable[[], 'Dependency']]:
+                kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     language = kwargs.get('language', 'c')
     if language not in {'c', 'cpp', 'fortran'}:
         # OpenMPI doesn't work without any other languages
         return []
 
-    candidates = []  # type: T.List[T.Callable[[], Dependency]]
+    candidates: 'TV_DepGenerators' = []
     compiler = detect_compiler('mpi', env, for_machine, language)  # type: T.Optional['CompilerType']
     if compiler is None:
         return []

mesonbuild/dependencies/scalapack.py

@@ -25,13 +25,13 @@ from .factory import factory_methods
 
 if T.TYPE_CHECKING:
     from ..environment import Environment, MachineChoice
-    from .base import DependencyType
+    from .factory import TV_DepGenerators
 
 
 @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CMAKE})
 def scalapack_factory(env: 'Environment', for_machine: 'MachineChoice',
                       kwargs: T.Dict[str, T.Any],
-                      methods: T.List[DependencyMethods]) -> T.List['DependencyType']:
+                      methods: T.List[DependencyMethods]) -> 'TV_DepGenerators':
     candidates = []
 
     if DependencyMethods.PKGCONFIG in methods:

run_mypy.py

@@ -18,6 +18,8 @@ modules = [
     # specific files
     'mesonbuild/arglist.py',
     # 'mesonbuild/coredata.py',
+    'mesonbuild/dependencies/__init__.py',
+    'mesonbuild/dependencies/factory.py',
     'mesonbuild/dependencies/base.py',
     'mesonbuild/dependencies/cmake.py',
     'mesonbuild/dependencies/coarrays.py',
@@ -25,6 +27,7 @@ modules = [
     'mesonbuild/dependencies/dub.py',
     'mesonbuild/dependencies/platform.py',
     'mesonbuild/dependencies/cuda.py',
+    'mesonbuild/dependencies/detect.py',
     'mesonbuild/dependencies/boost.py',
     'mesonbuild/dependencies/hdf5.py',
     'mesonbuild/dependencies/mpi.py',