Merge pull request #5276 from dcbaker/pkg-config-path-invalidate-cache

coredata: add pkg_config_path to depedency cache key

mesonbuild/coredata.py

@@ -27,6 +27,11 @@ import ast
 import argparse
 import configparser
 from typing import Optional, Any, TypeVar, Generic, Type, List, Union
+import typing
+import enum
+
+if typing.TYPE_CHECKING:
+    from . import dependencies
 
 version = '0.50.999'
 backendlist = ['ninja', 'vs', 'vs2010', 'vs2015', 'vs2017', 'vs2019', 'xcode']
@@ -221,6 +226,112 @@ def load_configs(filenames: List[str]) -> configparser.ConfigParser:
     return config
 
 
+if typing.TYPE_CHECKING:
+    CacheKeyType = typing.Tuple[typing.Tuple[typing.Any, ...], ...]
+    SubCacheKeyType = typing.Tuple[typing.Any, ...]
+
+
+class DependencyCacheType(enum.Enum):
+
+    OTHER = 0
+    PKG_CONFIG = 1
+
+    @classmethod
+    def from_type(cls, dep: 'dependencies.Dependency') -> 'DependencyCacheType':
+        from . import dependencies
+        # As more types gain search overrides they'll need to be added here
+        if isinstance(dep, dependencies.PkgConfigDependency):
+            return cls.PKG_CONFIG
+        return cls.OTHER
+
+
+class DependencySubCache:
+
+    def __init__(self, type_: DependencyCacheType):
+        self.types = [type_]
+        self.__cache = {}  # type: typing.Dict[SubCacheKeyType, dependencies.Dependency]
+
+    def __getitem__(self, key: 'SubCacheKeyType') -> 'dependencies.Dependency':
+        return self.__cache[key]
+
+    def __setitem__(self, key: 'SubCacheKeyType', value: 'dependencies.Dependency') -> None:
+        self.__cache[key] = value
+
+    def __contains__(self, key: 'SubCacheKeyType') -> bool:
+        return key in self.__cache
+
+    def values(self) -> typing.Iterable['dependencies.Dependency']:
+        return self.__cache.values()
+
+
+class DependencyCache:
+
+    """Class that stores a cache of dependencies.
+
+    This class is meant to encapsulate the fact that we need multiple keys to
+    successfully lookup by providing a simple get/put interface.
+    """
+
+    def __init__(self, builtins: typing.Dict[str, UserOption[typing.Any]], cross: bool):
+        self.__cache = OrderedDict()  # type: typing.MutableMapping[CacheKeyType, DependencySubCache]
+        self.__builtins = builtins
+        self.__is_cross = cross
+
+    def __calculate_subkey(self, type_: DependencyCacheType) -> typing.Tuple[typing.Any, ...]:
+        if type_ is DependencyCacheType.PKG_CONFIG:
+            if self.__is_cross:
+                return tuple(self.__builtins['cross_pkg_config_path'].value)
+            return tuple(self.__builtins['pkg_config_path'].value)
+        assert type_ is DependencyCacheType.OTHER, 'Someone forgot to update subkey calculations for a new type'
+        return tuple()
+
+    def __iter__(self) -> typing.Iterator['CacheKeyType']:
+        return self.keys()
+
+    def put(self, key: 'CacheKeyType', dep: 'dependencies.Dependency') -> None:
+        t = DependencyCacheType.from_type(dep)
+        if key not in self.__cache:
+            self.__cache[key] = DependencySubCache(t)
+        subkey = self.__calculate_subkey(t)
+        self.__cache[key][subkey] = dep
+
+    def get(self, key: 'CacheKeyType') -> typing.Optional['dependencies.Dependency']:
+        """Get a value from the cache.
+
+        If there is no cache entry then None will be returned.
+        """
+        try:
+            val = self.__cache[key]
+        except KeyError:
+            return None
+
+        for t in val.types:
+            subkey = self.__calculate_subkey(t)
+            try:
+                return val[subkey]
+            except KeyError:
+                pass
+        return None
+
+    def values(self) -> typing.Iterator['dependencies.Dependency']:
+        for c in self.__cache.values():
+            yield from c.values()
+
+    def keys(self) -> typing.Iterator['CacheKeyType']:
+        return iter(self.__cache.keys())
+
+    def items(self) -> typing.Iterator[typing.Tuple['CacheKeyType', typing.List['dependencies.Dependency']]]:
+        for k, v in self.__cache.items():
+            vs = []
+            for t in v.types:
+                subkey = self.__calculate_subkey(t)
+                if subkey in v:
+                    vs.append(v[subkey])
+            yield k, vs
+
+    def clear(self) -> None:
+        self.__cache.clear()
+
 # This class contains all data that must persist over multiple
 # invocations of Meson. It is roughly the same thing as
 # cmakecache.
@@ -248,7 +359,14 @@ class CoreData:
         self.cross_files = self.__load_config_files(options.cross_file, 'cross')
         self.compilers = OrderedDict()
         self.cross_compilers = OrderedDict()
-        self.deps = OrderedDict()
+
+        build_cache = DependencyCache(self.builtins, False)
+        if self.cross_files:
+            host_cache = DependencyCache(self.builtins, True)
+        else:
+            host_cache = build_cache
+        self.deps = PerMachine(build_cache, host_cache)  # type: PerMachine[DependencyCache]
+
         self.compiler_check_cache = OrderedDict()
         # Only to print a warning if it changes between Meson invocations.
         self.config_files = self.__load_config_files(options.native_file, 'native')
@@ -510,8 +628,11 @@ class CoreData:
 
         # Some options default to environment variables if they are
         # unset, set those now. These will either be overwritten
-        # below, or they won't.
-        options['pkg_config_path'] = os.environ.get('PKG_CONFIG_PATH', '').split(':')
+        # below, or they won't. These should only be set on the first run.
+        if env.first_invocation:
+            p_env = os.environ.get('PKG_CONFIG_PATH')
+            if p_env:
+                options['pkg_config_path'] = p_env.split(':')
 
         for k, v in env.cmd_line_options.items():
             if subproject:

mesonbuild/dependencies/base.py

@@ -2358,8 +2358,8 @@ class ExtraFrameworkDependency(ExternalDependency):
         return 'framework'
 
 
-def get_dep_identifier(name, kwargs, want_cross: bool) -> Tuple:
-    identifier = (name, want_cross)
+def get_dep_identifier(name, kwargs) -> Tuple:
+    identifier = (name, )
     for key, value in kwargs.items():
         # 'version' is irrelevant for caching; the caller must check version matches
         # 'native' is handled above with `want_cross`

mesonbuild/environment.py

@@ -535,7 +535,7 @@ class Environment:
         self.coredata.meson_command = mesonlib.meson_command
         self.first_invocation = True
 
-    def is_cross_build(self):
+    def is_cross_build(self) -> bool:
         return not self.machines.matches_build_machine(MachineChoice.HOST)
 
     def dump_coredata(self):

mesonbuild/interpreter.py

@@ -2859,14 +2859,13 @@ external dependencies (including libraries) must go to "dependencies".''')
         # FIXME: Not all dependencies support such a distinction right now,
         # and we repeat this check inside dependencies that do. We need to
         # consolidate this somehow.
-        is_cross = self.environment.is_cross_build()
-        if 'native' in kwargs and is_cross:
-            want_cross = not kwargs['native']
+        if self.environment.is_cross_build() and kwargs.get('native', False):
+            for_machine = MachineChoice.BUILD
         else:
-            want_cross = is_cross
+            for_machine = MachineChoice.HOST
 
-        identifier = dependencies.get_dep_identifier(name, kwargs, want_cross)
-        cached_dep = self.coredata.deps.get(identifier)
+        identifier = dependencies.get_dep_identifier(name, kwargs)
+        cached_dep = self.coredata.deps[for_machine].get(identifier)
         if cached_dep:
             if not cached_dep.found():
                 mlog.log('Dependency', mlog.bold(name),
@@ -3018,7 +3017,11 @@ external dependencies (including libraries) must go to "dependencies".''')
             # cannot cache them. They must always be evaluated else
             # we won't actually read all the build files.
             if dep.found():
-                self.coredata.deps[identifier] = dep
+                if self.environment.is_cross_build() and kwargs.get('native', False):
+                    for_machine = MachineChoice.BUILD
+                else:
+                    for_machine = MachineChoice.HOST
+                self.coredata.deps[for_machine].put(identifier, dep)
                 return DependencyHolder(dep, self.subproject)
 
         if has_fallback:

mesonbuild/mconf.py

@@ -63,7 +63,8 @@ class Conf:
             raise ConfException('Directory {} is neither a Meson build directory nor a project source directory.'.format(build_dir))
 
     def clear_cache(self):
-        self.coredata.deps = {}
+        self.coredata.deps.host.clear()
+        self.coredata.deps.build.clear()
 
     def set_options(self, options):
         self.coredata.set_options(options)

mesonbuild/mintro.py

@@ -282,7 +282,7 @@ def list_deps_from_source(intr: IntrospectionInterpreter):
 
 def list_deps(coredata: cdata.CoreData):
     result = []
-    for d in coredata.deps.values():
+    for d in coredata.deps.host.values():
         if d.found():
             result += [{'name': d.name,
                         'compile_args': d.get_compile_args(),

mesonbuild/modules/rpm.py

@@ -83,7 +83,7 @@ class RPMModule(ExtensionModule):
             fn.write('BuildRequires: meson\n')
             for compiler in required_compilers:
                 fn.write('BuildRequires: %s\n' % compiler)
-            for dep in coredata.environment.coredata.deps:
+            for dep in coredata.environment.coredata.deps.host:
                 fn.write('BuildRequires: pkgconfig(%s)\n' % dep[0])
 #   ext_libs and ext_progs have been removed from coredata so the following code
 #   no longer works. It is kept as a reminder of the idea should anyone wish

mesonbuild/munstable_coredata.py

@@ -97,13 +97,11 @@ def run(options):
             print('Cached cross compilers:')
             dump_compilers(v)
         elif k == 'deps':
-            native = []
-            cross = []
-            for dep_key, dep in sorted(v.items()):
-                if dep_key[1]:
-                    cross.append((dep_key, dep))
-                else:
-                    native.append((dep_key, dep))
+            native = list(sorted(v.build.items()))
+            if v.host is not v.build:
+                cross = list(sorted(v.host.items()))
+            else:
+                cross = []
 
             def print_dep(dep_key, dep):
                 print('  ' + dep_key[0] + ": ")
@@ -115,12 +113,14 @@ def run(options):
 
             if native:
                 print('Cached native dependencies:')
-                for dep_key, dep in native:
-                    print_dep(dep_key, dep)
+                for dep_key, deps in native:
+                    for dep in deps:
+                        print_dep(dep_key, dep)
             if cross:
                 print('Cached dependencies:')
-                for dep_key, dep in cross:
-                    print_dep(dep_key, dep)
+                for dep_key, deps in cross:
+                    for dep in deps:
+                        print_dep(dep_key, dep)
         else:
             print(k + ':')
             print(textwrap.indent(pprint.pformat(v), '  '))