coredata: Correctly handle receiving a pipe for native/cross files

* coredata: Correctly handle receiving a pipe for native/cross files

In some cases a cross/native file may be a pipe, such as when using bash
process replacement `meson --native-file
<([binaries]llvm-config='/opt/bin/llvm-config')`, for example. In this
case we copy the contents of the pipe into a file in the meson-private
directory so we can create a proper ninja dependency, and be able to
reload the file on --wipe/--reconfigure. This requires some extra
negotiation to preserve these native/cross files.

Fixes #5505

* run_unitests: Add a unit test for native files that are pipes

Using mkfifo.

mesonbuild/coredata.py

@@ -346,7 +346,7 @@ _V = TypeVar('_V')
 
 class CoreData:
 
-    def __init__(self, options):
+    def __init__(self, options: argparse.Namespace, scratch_dir: str):
         self.lang_guids = {
             'default': '8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942',
             'c': '8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942',
@@ -364,7 +364,7 @@ class CoreData:
         self.user_options = {} # : Dict[str, UserOption]
         self.compiler_options = PerMachine({}, {})
         self.base_options = {} # : Dict[str, UserOption]
-        self.cross_files = self.__load_config_files(options.cross_file, 'cross')
+        self.cross_files = self.__load_config_files(options, scratch_dir, 'cross')
         self.compilers = PerMachine(OrderedDict(), OrderedDict())
 
         build_cache = DependencyCache(self.builtins_per_machine, MachineChoice.BUILD)
@@ -372,21 +372,42 @@ class CoreData:
         self.deps = PerMachine(build_cache, host_cache)  # type: PerMachine[DependencyCache]
         self.compiler_check_cache = OrderedDict()
         # Only to print a warning if it changes between Meson invocations.
-        self.config_files = self.__load_config_files(options.native_file, 'native')
+        self.config_files = self.__load_config_files(options, scratch_dir, 'native')
         self.libdir_cross_fixup()
 
     @staticmethod
-    def __load_config_files(filenames: Optional[List[str]], ftype: str) -> List[str]:
+    def __load_config_files(options: argparse.Namespace, scratch_dir: str, ftype: str) -> List[str]:
         # Need to try and make the passed filenames absolute because when the
         # files are parsed later we'll have chdir()d.
+        if ftype == 'cross':
+            filenames = options.cross_file
+        else:
+            filenames = options.native_file
+
         if not filenames:
             return []
 
         real = []
-        for f in filenames:
+        for i, f in enumerate(filenames):
             f = os.path.expanduser(os.path.expandvars(f))
             if os.path.exists(f):
-                real.append(os.path.abspath(f))
+                if os.path.isfile(f):
+                    real.append(os.path.abspath(f))
+                elif os.path.isdir(f):
+                    raise MesonException('Cross and native files must not be directories')
+                else:
+                    # in this case we've been passed some kind of pipe, copy
+                    # the contents of that file into the meson private (scratch)
+                    # directory so that it can be re-read when wiping/reconfiguring
+                    copy = os.path.join(scratch_dir, '{}.{}.ini'.format(uuid.uuid4(), ftype))
+                    with open(f, 'r') as rf:
+                        with open(copy, 'w') as wf:
+                            wf.write(rf.read())
+                    real.append(copy)
+
+                    # Also replace the command line argument, as the pipe
+                    # probably wont exist on reconfigure
+                    filenames[i] = copy
                 continue
             elif sys.platform != 'win32':
                 paths = [

mesonbuild/environment.py

@@ -403,6 +403,7 @@ class Environment:
                     raise e
         else:
             # Just create a fresh coredata in this case
+            self.scratch_dir = ''
             self.create_new_coredata(options)
 
         ## locally bind some unfrozen configuration
@@ -514,7 +515,7 @@ class Environment:
         # WARNING: Don't use any values from coredata in __init__. It gets
         # re-initialized with project options by the interpreter during
         # build file parsing.
-        self.coredata = coredata.CoreData(options)
+        self.coredata = coredata.CoreData(options, self.scratch_dir)
         # Used by the regenchecker script, which runs meson
         self.coredata.meson_command = mesonlib.meson_command
         self.first_invocation = True

mesonbuild/msetup.py

@@ -19,6 +19,9 @@ import os.path
 import platform
 import cProfile as profile
 import argparse
+import tempfile
+import shutil
+import glob
 
 from . import environment, interpreter, mesonlib
 from . import build
@@ -60,38 +63,36 @@ class MesonApp:
                                                                options.sourcedir,
                                                                options.reconfigure,
                                                                options.wipe)
-
         if options.wipe:
             # Make a copy of the cmd line file to make sure we can always
             # restore that file if anything bad happens. For example if
             # configuration fails we need to be able to wipe again.
-            filename = coredata.get_cmd_line_file(self.build_dir)
-            try:
-                with open(filename, 'r') as f:
-                    content = f.read()
-            except FileNotFoundError:
-                raise MesonException(
-                    'Cannot find cmd_line.txt. This is probably because this '
-                    'build directory was configured with a meson version < 0.49.0.')
-
-            coredata.read_cmd_line_file(self.build_dir, options)
-
-            try:
-                # Don't delete the whole tree, just all of the files and
-                # folders in the tree. Otherwise calling wipe form the builddir
-                # will cause a crash
-                for l in os.listdir(self.build_dir):
-                    l = os.path.join(self.build_dir, l)
-                    if os.path.isdir(l):
-                        mesonlib.windows_proof_rmtree(l)
-                    else:
-                        mesonlib.windows_proof_rm(l)
-            finally:
-                # Restore the file
-                path = os.path.dirname(filename)
-                os.makedirs(path, exist_ok=True)
-                with open(filename, 'w') as f:
-                    f.write(content)
+            restore = []
+            with tempfile.TemporaryDirectory() as d:
+                for filename in [coredata.get_cmd_line_file(self.build_dir)] + glob.glob(os.path.join(self.build_dir, environment.Environment.private_dir, '*.ini')):
+                    try:
+                        restore.append((shutil.copy(filename, d), filename))
+                    except FileNotFoundError:
+                        raise MesonException(
+                            'Cannot find cmd_line.txt. This is probably because this '
+                            'build directory was configured with a meson version < 0.49.0.')
+
+                coredata.read_cmd_line_file(self.build_dir, options)
+
+                try:
+                    # Don't delete the whole tree, just all of the files and
+                    # folders in the tree. Otherwise calling wipe form the builddir
+                    # will cause a crash
+                    for l in os.listdir(self.build_dir):
+                        l = os.path.join(self.build_dir, l)
+                        if os.path.isdir(l):
+                            mesonlib.windows_proof_rmtree(l)
+                        else:
+                            mesonlib.windows_proof_rm(l)
+                finally:
+                    for b, f in restore:
+                        os.makedirs(os.path.dirname(f), exist_ok=True)
+                        shutil.move(b, f)
 
         self.options = options
 

run_unittests.py

@@ -29,6 +29,7 @@ import pickle
 import functools
 import io
 import operator
+import threading
 from itertools import chain
 from unittest import mock
 from configparser import ConfigParser
@@ -5742,9 +5743,9 @@ class NativeFileTests(BasePlatformTests):
                     f.write("{}='{}'\n".format(k, v))
         return filename
 
-    def helper_create_binary_wrapper(self, binary, **kwargs):
+    def helper_create_binary_wrapper(self, binary, dir_=None, **kwargs):
         """Creates a wrapper around a binary that overrides specific values."""
-        filename = os.path.join(self.builddir, 'binary_wrapper{}.py'.format(self.current_wrapper))
+        filename = os.path.join(dir_ or self.builddir, 'binary_wrapper{}.py'.format(self.current_wrapper))
         self.current_wrapper += 1
         if is_haiku():
             chbang = '#!/bin/env python3'
@@ -5818,6 +5819,28 @@ class NativeFileTests(BasePlatformTests):
             '--native-file', config, '--native-file', config2,
             '-Dcase=find_program'])
 
+    @unittest.skipIf(os.name != 'posix', 'Uses fifos, which are not available on non Unix OSes.')
+    def test_native_file_is_pipe(self):
+        fifo = os.path.join(self.builddir, 'native.file')
+        os.mkfifo(fifo)
+        with tempfile.TemporaryDirectory() as d:
+            wrapper = self.helper_create_binary_wrapper('bash', d, version='12345')
+
+            def filler():
+                with open(fifo, 'w') as f:
+                    f.write('[binaries]\n')
+                    f.write("bash = '{}'\n".format(wrapper))
+
+            thread = threading.Thread(target=filler)
+            thread.start()
+
+            self.init(self.testcase, extra_args=['--native-file', fifo, '-Dcase=find_program'])
+
+            thread.join()
+            os.unlink(fifo)
+
+            self.init(self.testcase, extra_args=['--wipe'])
+
     def test_multiple_native_files(self):
         wrapper = self.helper_create_binary_wrapper('bash', version='12345')
         config = self.helper_create_native_file({'binaries': {'bash': wrapper}})