coredata: store cross/native files in the same form they will be used (#5224)

* coredata: store cross/native files in the same form they will be used

Currently they're forced to absolute paths when they're stored in the
coredata datastructure, then when they're loaded we de-absolute path
them to check if they're in the system wide directories. This doesn't
work at all, since the ninja backend will generat a dependency on a
file that is in the source directory unless the path was already given
as absolute. This results in builds being retriggereed forever due to
a non-existant file.

The right way to do this is to figure out whether the file is in the
build directory, is absolute, or is in one of the system paths at
creation time, and store that path as absolute. Then the code that
reads the file and the code that generates the dependencies in the
ninja backend just takes the computed list and there is no mismatch
between them.

Fixes #5257

* run_unittests: Add a test for correct native file storage

This tests the bug in #5257

mesonbuild/coredata.py

@@ -26,7 +26,7 @@ from .wrap import WrapMode
 import ast
 import argparse
 import configparser
-from typing import Optional, Any, TypeVar, Generic, Type
+from typing import Optional, Any, TypeVar, Generic, Type, List
 
 version = '0.50.999'
 backendlist = ['ninja', 'vs', 'vs2010', 'vs2015', 'vs2017', 'vs2019', 'xcode']
@@ -212,32 +212,10 @@ class UserFeatureOption(UserComboOption):
         return self.value == 'auto'
 
 
-def load_configs(filenames, subdir):
+def load_configs(filenames: List[str]) -> configparser.ConfigParser:
     """Load configuration files from a named subdirectory."""
-    def gen():
-        for f in filenames:
-            f = os.path.expanduser(os.path.expandvars(f))
-            if os.path.exists(f):
-                yield f
-                continue
-            elif sys.platform != 'win32':
-                f = os.path.basename(f)
-                paths = [
-                    os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')),
-                ] + os.environ.get('XDG_DATA_DIRS', '/usr/local/share:/usr/share').split(':')
-                for path in paths:
-                    path_to_try = os.path.join(path, 'meson', subdir, f)
-                    if os.path.isfile(path_to_try):
-                        yield path_to_try
-                        break
-                else:
-                    raise MesonException('Cannot find specified native file: ' + f)
-                continue
-
-            raise MesonException('Cannot find specified native file: ' + f)
-
     config = configparser.ConfigParser()
-    config.read(gen())
+    config.read(filenames)
     return config
 
 
@@ -265,23 +243,42 @@ class CoreData:
         self.user_options = {}
         self.compiler_options = PerMachine({}, {}, {})
         self.base_options = {}
-        self.cross_files = self.__load_config_files(options.cross_file)
+        self.cross_files = self.__load_config_files(options.cross_file, 'cross')
         self.compilers = OrderedDict()
         self.cross_compilers = OrderedDict()
         self.deps = OrderedDict()
         # Only to print a warning if it changes between Meson invocations.
-        self.config_files = self.__load_config_files(options.native_file)
+        self.config_files = self.__load_config_files(options.native_file, 'native')
         self.libdir_cross_fixup()
 
     @staticmethod
-    def __load_config_files(filenames):
+    def __load_config_files(filenames: Optional[List[str]], ftype: str) -> List[str]:
         # Need to try and make the passed filenames absolute because when the
         # files are parsed later we'll have chdir()d.
         if not filenames:
             return []
-        filenames = [os.path.abspath(os.path.expanduser(os.path.expanduser(f)))
+
-                     for f in filenames]
+        real = []
-        return filenames
+        for f in filenames:
+            f = os.path.expanduser(os.path.expandvars(f))
+            if os.path.exists(f):
+                real.append(os.path.abspath(f))
+                continue
+            elif sys.platform != 'win32':
+                paths = [
+                    os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')),
+                ] + os.environ.get('XDG_DATA_DIRS', '/usr/local/share:/usr/share').split(':')
+                for path in paths:
+                    path_to_try = os.path.join(path, 'meson', ftype, f)
+                    if os.path.isfile(path_to_try):
+                        real.append(path_to_try)
+                        break
+                else:
+                    raise MesonException('Cannot find specified {} file: {}'.format(ftype, f))
+                continue
+
+            raise MesonException('Cannot find specified {} file: {}'.format(ftype, f))
+        return real
 
     def libdir_cross_fixup(self):
         # By default set libdir to "lib" when cross compiling since

mesonbuild/environment.py

@@ -422,13 +422,13 @@ class Environment:
 
         if self.coredata.config_files is not None:
             config = MesonConfigFile.from_config_parser(
-                coredata.load_configs(self.coredata.config_files, 'native'))
+                coredata.load_configs(self.coredata.config_files))
             self.binaries.build = BinaryTable(config.get('binaries', {}))
             self.paths.build = Directories(**config.get('paths', {}))
 
         if self.coredata.cross_files:
             config = MesonConfigFile.from_config_parser(
-                coredata.load_configs(self.coredata.cross_files, 'cross'))
+                coredata.load_configs(self.coredata.cross_files))
             self.properties.host = Properties(config.get('properties', {}), False)
             self.binaries.host = BinaryTable(config.get('binaries', {}), False)
             if 'host_machine' in config:

run_unittests.py

@@ -5911,6 +5911,21 @@ class NativeFileTests(BasePlatformTests):
                   extra_args=['--native-file', os.path.join(testcase, 'nativefile'),
                               '-Ddef_libdir=liblib', '-Dlibdir=liblib'])
 
+    def test_compile_sys_path(self):
+        """Compiling with a native file stored in a system path works.
+
+        There was a bug which caused the paths to be stored incorrectly and
+        would result in ninja invoking meson in an infinite loop. This tests
+        for that by actually invoking ninja.
+        """
+        testcase = os.path.join(self.common_test_dir, '1 trivial')
+
+        # It really doesn't matter what's in the native file, just that it exists
+        config = self.helper_create_native_file({'binaries': {'bash': 'false'}})
+
+        self.init(testcase, extra_args=['--native-file', config])
+        self.build()
+
 
 class CrossFileTests(BasePlatformTests):