Merge pull request #4216 from dcbaker/wip/config-file

native file support

docs/markdown/Native-environments.md

@@ -0,0 +1,76 @@
+---
+short-description: Setting up native compilation
+...
+
+# Persistent native environments
+
+New in 0.49.0
+
+Meson has [cross files for describing cross compilation environments](Cross-compilation.md),
+for describing native environments it has equivalent "native files".
+
+Natives describe the *build machine*, and can be used to override properties of
+non-cross builds, as well as properties that are marked as "native" in a cross
+build.
+
+There are a couple of reasons you might want to use a native file to keep a
+persistent environment:
+
+* To build with a non-default native tool chain (such as clang instead of gcc)
+* To use a non-default version of another binary, such as yacc, or llvm-config
+
+
+## Changing native file settings
+
+All of the rules about cross files and changed settings apply to native files
+as well, see [here](Cross-compilation.md#Changing-cross-file-settings)
+
+
+## Defining the environment
+
+### Binaries
+
+Currently the only use of native files is to override native binaries. This
+includes the compilers and binaries collected with `find_program`, and those
+used by dependencies that use a config-tool instead of pkgconfig for detection,
+like `llvm-config`
+
+```ini
+[binaries]
+c = '/usr/local/bin/clang'
+cpp = '/usr/local/bin/clang++'
+rust = '/usr/local/bin/rust'
+llvm-conifg = '/usr/local/llvm-svn/bin/llvm-config'
+```
+
+## Loading multiple native files
+
+Unlike cross file, native files allow layering. More than one native file can be
+loaded, with values from a previous file being overridden by the next. The
+intention of this is not overriding, but to allow composing native files.
+
+For example, if there is a project using C and C++, python 3.4-3.7, and LLVM
+5-7, and it needs to build with clang 5, 6, and 7, and gcc 5.x, 6.x, and 7.x;
+expressing all of these configurations in monolithic configurations would
+result in 81 different native files. By layering them, it can be expressed by
+just 12 native files.
+
+
+## Native file locations
+
+Like cross files, native files may be installed to user or system wide
+locations, defined as:
+  - $XDG_DATA_DIRS/meson/native 
+    (/usr/local/share/meson/native:/usr/share/meson/native if $XDG_DATA_DIRS is
+    undefined)
+  - $XDG_DATA_HOME/meson/native ($HOME/.local/share/meson/native if
+    $XDG_DATA_HOME is undefined)
+
+The order of locations tried is as follows:
+ - A file relative to the local dir
+ - The user local location
+ - The system wide locations in order
+
+These files are not intended to be shipped by distributions, unless they are
+specifically for distribution packaging, they are mainly intended for
+developers.

docs/markdown/snippets/native_files.md

@@ -0,0 +1,15 @@
+## Native config files
+
+Native files are the counterpart to cross files, and allow specifying
+information about the build machine, both when cross compiling and when not.
+
+Currently the native files only allow specifying the names of binaries, similar
+to the cross file, for example:
+
+```ini
+[binaries]
+llvm-config = "/opt/llvm-custom/bin/llvm-config"
+```
+
+Will override the llvm-config used for *native* binaries. Targets for the host
+machine will continue to use the cross file.

docs/sitemap.txt

@@ -9,6 +9,7 @@ index.md
 		Using-with-Visual-Studio.md
 		Meson-sample.md
 		Syntax.md
+		Native-environments.md
 		Build-targets.md
 		Include-directories.md
 		Installing.md

mesonbuild/coredata.py

@@ -183,6 +183,7 @@ class UserArrayOption(UserOption):
                     ', '.join(bad), ', '.join(self.choices)))
         return newvalue
 
+
 class UserFeatureOption(UserComboOption):
     static_choices = ['enabled', 'disabled', 'auto']
 
@@ -198,6 +199,72 @@ class UserFeatureOption(UserComboOption):
     def is_auto(self):
         return self.value == 'auto'
 
+
+def load_configs(filenames):
+    """Load native files."""
+    def gen():
+        for f in filenames:
+            f = os.path.expanduser(os.path.expandvars(f))
+            if os.path.exists(f):
+                yield f
+                continue
+            elif sys.platform != 'win32':
+                f = os.path.basename(f)
+                paths = [
+                    os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share')),
+                ] + os.environ.get('XDG_DATA_DIRS', '/usr/local/share:/usr/share').split(':')
+                for path in paths:
+                    path_to_try = os.path.join(path, 'meson', 'native', f)
+                    if os.path.isfile(path_to_try):
+                        yield path_to_try
+                        break
+                else:
+                    raise MesonException('Cannot find specified native file: ' + f)
+                continue
+
+            raise MesonException('Cannot find specified native file: ' + f)
+
+    config = configparser.SafeConfigParser()
+    config.read(gen())
+    return config
+
+
+def _get_section(config, section):
+    if config.has_section(section):
+        final = {}
+        for k, v in config.items(section):
+            # Windows paths...
+            v = v.replace('\\', '\\\\')
+            try:
+                final[k] = ast.literal_eval(v)
+            except SyntaxError:
+                raise MesonException(
+                    'Malformed value in native file variable: {}'.format(v))
+        return final
+    return {}
+
+
+class ConfigData:
+
+    """Contains configuration information provided by the user for the build."""
+
+    def __init__(self, config=None):
+        if config:
+            self.binaries = _get_section(config, 'binaries')
+            # global is a keyword and globals is a builtin, rather than mangle it,
+            # use a similar word
+            self.universal = _get_section(config, 'globals')
+            self.subprojects = {s: _get_section(config, s) for s in config.sections()
+                                if s not in {'binaries', 'globals'}}
+        else:
+            self.binaries = {}
+            self.universal = {}
+            self.subprojects = {}
+
+    def get_binaries(self, name):
+        return self.binaries.get(name, None)
+
+
 # This class contains all data that must persist over multiple
 # invocations of Meson. It is roughly the same thing as
 # cmakecache.
@@ -229,6 +296,15 @@ class CoreData:
         self.deps = OrderedDict()
         # Only to print a warning if it changes between Meson invocations.
         self.pkgconf_envvar = os.environ.get('PKG_CONFIG_PATH', '')
+        self.config_files = self.__load_config_files(options.native_file)
+
+    @staticmethod
+    def __load_config_files(filenames):
+        if not filenames:
+            return []
+        filenames = [os.path.abspath(os.path.expanduser(os.path.expanduser(f)))
+                     for f in filenames]
+        return filenames
 
     @staticmethod
     def __load_cross_file(filename):

mesonbuild/dependencies/base.py

@@ -399,6 +399,8 @@ class ConfigToolDependency(ExternalDependency):
                              'Falling back to searching PATH. This may find a '
                              'native version of {0}!'.format(self.tool_name))
                 tools = self.tools
+        elif self.tool_name in self.env.config_info.binaries:
+            tools = [self.env.config_info.binaries[self.tool_name]]
         else:
             tools = self.tools
 
@@ -500,7 +502,8 @@ class PkgConfigDependency(ExternalDependency):
                 if self.required:
                     raise DependencyException('Pkg-config binary missing from cross file')
             else:
-                potential_pkgbin = ExternalProgram.from_cross_info(environment.cross_info, 'pkgconfig')
+                potential_pkgbin = ExternalProgram.from_bin_list(
+                    environment.cross_info.config['binaries'], 'pkgconfig')
                 if potential_pkgbin.found():
                     self.pkgbin = potential_pkgbin
                 else:
@@ -1076,10 +1079,10 @@ class ExternalProgram:
         return ' '.join(self.command)
 
     @staticmethod
-    def from_cross_info(cross_info, name):
-        if name not in cross_info.config['binaries']:
+    def from_bin_list(bins, name):
+        if name not in bins:
             return NonExistingExternalProgram()
-        command = cross_info.config['binaries'][name]
+        command = bins[name]
         if not isinstance(command, (list, str)):
             raise MesonException('Invalid type {!r} for binary {!r} in cross file'
                                  ''.format(command, name))
@@ -1238,8 +1241,8 @@ class ExternalProgram:
 class NonExistingExternalProgram(ExternalProgram):
     "A program that will never exist"
 
-    def __init__(self):
-        self.name = 'nonexistingprogram'
+    def __init__(self, name='nonexistingprogram'):
+        self.name = name
         self.command = [None]
         self.path = None
 

mesonbuild/dependencies/ui.py

@@ -30,7 +30,7 @@ from ..mesonlib import (
 from ..environment import detect_cpu
 
 from .base import DependencyException, DependencyMethods
-from .base import ExternalDependency, ExternalProgram
+from .base import ExternalDependency, ExternalProgram, NonExistingExternalProgram
 from .base import ExtraFrameworkDependency, PkgConfigDependency
 from .base import ConfigToolDependency
 
@@ -230,21 +230,46 @@ class QtBaseDependency(ExternalDependency):
             self.from_text = mlog.format_list(methods)
             self.version = None
 
-    def compilers_detect(self):
+    def compilers_detect(self, interp_obj):
         "Detect Qt (4 or 5) moc, uic, rcc in the specified bindir or in PATH"
-        if self.bindir or for_windows(self.env.is_cross_build(), self.env):
-            moc = ExternalProgram(os.path.join(self.bindir, 'moc'), silent=True)
-            uic = ExternalProgram(os.path.join(self.bindir, 'uic'), silent=True)
-            rcc = ExternalProgram(os.path.join(self.bindir, 'rcc'), silent=True)
-            lrelease = ExternalProgram(os.path.join(self.bindir, 'lrelease'), silent=True)
-        else:
-            # We don't accept unsuffixed 'moc', 'uic', and 'rcc' because they
-            # are sometimes older, or newer versions.
-            moc = ExternalProgram('moc-' + self.name, silent=True)
-            uic = ExternalProgram('uic-' + self.name, silent=True)
-            rcc = ExternalProgram('rcc-' + self.name, silent=True)
-            lrelease = ExternalProgram('lrelease-' + self.name, silent=True)
-        return moc, uic, rcc, lrelease
+        # It is important that this list does not change order as the order of
+        # the returned ExternalPrograms will change as well
+        bins = ['moc', 'uic', 'rcc', 'lrelease']
+        found = {b: NonExistingExternalProgram(name='{}-{}'.format(b, self.name))
+                 for b in bins}
+
+        def gen_bins():
+            for b in bins:
+                yield '{}-{}'.format(b, self.name), b, False
+                yield b, b, self.required
+
+        for b, name, required in gen_bins():
+            if found[name].found():
+                continue
+
+            # prefer the <tool>-qt<version> of the tool to the plain one, as we
+            # don't know what the unsuffixed one points to without calling it.
+            p = interp_obj.find_program_impl([b], silent=True, required=required).held_object
+            if not p.found():
+                continue
+
+            if b.startswith('lrelease'):
+                arg = ['-version']
+            elif mesonlib.version_compare(self.version, '>= 5'):
+                arg = ['--version']
+            else:
+                arg = ['-v']
+
+            # Ensure that the version of qt and each tool are the same
+            _, out, err = mesonlib.Popen_safe(p.get_command() + arg)
+            if b.startswith('lrelease') or not self.version.startswith('4'):
+                care = out
+            else:
+                care = err
+            if mesonlib.version_compare(self.version, '== {}'.format(care.split(' ')[-1])):
+                found[name] = p
+
+        return tuple([found[b] for b in bins])
 
     def _pkgconfig_detect(self, mods, kwargs):
         # We set the value of required to False so that we can try the
@@ -302,8 +327,15 @@ class QtBaseDependency(ExternalDependency):
     def _find_qmake(self, qmake):
         # Even when cross-compiling, if a cross-info qmake is not specified, we
         # fallback to using the qmake in PATH because that's what we used to do
-        if self.env.is_cross_build() and 'qmake' in self.env.cross_info.config['binaries']:
-            return ExternalProgram.from_cross_info(self.env.cross_info, 'qmake')
+        if self.env.is_cross_build():
+            if 'qmake' in self.env.cross_info.config['binaries']:
+                return ExternalProgram.from_bin_list(self.env.cross_info.config['binaries'], 'qmake')
+        elif self.env.config_info:
+            # Prefer suffixed to unsuffixed version
+            p = ExternalProgram.from_bin_list(self.env.config_info.binaries, 'qmake-' + self.name)
+            if p.found():
+                return p
+            return ExternalProgram.from_bin_list(self.env.config_info.binaries, 'qmake')
         return ExternalProgram(qmake, silent=True)
 
     def _qmake_detect(self, mods, kwargs):

mesonbuild/environment.py

@@ -340,7 +340,8 @@ class Environment:
             self.cross_info = CrossBuildInfo(self.coredata.cross_file)
             if 'exe_wrapper' in self.cross_info.config['binaries']:
                 from .dependencies import ExternalProgram
-                self.exe_wrapper = ExternalProgram.from_cross_info(self.cross_info, 'exe_wrapper')
+                self.exe_wrapper = ExternalProgram.from_bin_list(
+                    self.cross_info.config['binaries'], 'exe_wrapper')
             if 'host_machine' in self.cross_info.config:
                 self.machines.host = MachineInfo.from_literal(
                     self.cross_info.config['host_machine'])
@@ -351,6 +352,12 @@ class Environment:
             self.cross_info = None
         self.machines.default_missing()
 
+        if self.coredata.config_files:
+            self.config_info = coredata.ConfigData(
+                coredata.load_configs(self.coredata.config_files))
+        else:
+            self.config_info = coredata.ConfigData()
+
         self.cmd_line_options = options.cmd_line_options.copy()
 
         # List of potential compilers.
@@ -505,7 +512,10 @@ class Environment:
         The list of compilers is detected in the exact same way for
         C, C++, ObjC, ObjC++, Fortran, CS so consolidate it here.
         '''
+        is_cross = False
+        exe_wrap = None
         evar = BinaryTable.evarMap[lang]
+
         if self.is_cross_build() and want_cross:
             if lang not in self.cross_info.config['binaries']:
                 raise EnvironmentException('{!r} compiler binary not defined in cross file'.format(lang))
@@ -521,13 +531,13 @@ class Environment:
                 shlex.split(os.environ[evar]))
             # Return value has to be a list of compiler 'choices'
             compilers = [compilers]
-            is_cross = False
-            exe_wrap = None
+        elif lang in self.config_info.binaries:
+            compilers, ccache = BinaryTable.parse_entry(
+                mesonlib.stringlistify(self.config_info.binaries[lang]))
+            compilers = [compilers]
         else:
             compilers = getattr(self, 'default_' + lang)
             ccache = BinaryTable.detect_ccache()
-            is_cross = False
-            exe_wrap = None
         return compilers, ccache, is_cross, exe_wrap
 
     def _handle_exceptions(self, exceptions, binaries, bintype='compiler'):
@@ -798,7 +808,11 @@ class Environment:
         self._handle_exceptions(popen_exceptions, compilers)
 
     def detect_java_compiler(self):
-        exelist = ['javac']
+        if 'java' in self.config_info.binaries:
+            exelist = mesonlib.stringlistify(self.config_info.binaries['java'])
+        else:
+            exelist = ['javac']
+
         try:
             p, out, err = Popen_safe(exelist + ['-version'])
         except OSError:
@@ -831,6 +845,8 @@ class Environment:
     def detect_vala_compiler(self):
         if 'VALAC' in os.environ:
             exelist = shlex.split(os.environ['VALAC'])
+        elif 'vala' in self.config_info.binaries:
+            exelist = mesonlib.stringlistify(self.config_info.binaries['vala'])
         else:
             exelist = ['valac']
         try:
@@ -875,6 +891,8 @@ class Environment:
         elif self.is_cross_build() and want_cross:
             exelist = mesonlib.stringlistify(self.cross_info.config['binaries']['d'])
             is_cross = True
+        elif 'd' in self.config_info.binaries:
+            exelist = mesonlib.stringlistify(self.config_info.binaries['d'])
         elif shutil.which("ldc2"):
             exelist = ['ldc2']
         elif shutil.which("ldc"):
@@ -912,7 +930,10 @@ class Environment:
         raise EnvironmentException('Unknown compiler "' + ' '.join(exelist) + '"')
 
     def detect_swift_compiler(self):
-        exelist = ['swiftc']
+        if 'swift' in self.config_info.binaries:
+            exelist = mesonlib.stringlistify(self.config_info.binaries['swift'])
+        else:
+            exelist = ['swiftc']
         try:
             p, _, err = Popen_safe(exelist + ['-v'])
         except OSError:

mesonbuild/interpreter.py

@@ -2748,8 +2748,7 @@ external dependencies (including libraries) must go to "dependencies".''')
             self.coredata. base_options[optname] = oobj
         self.emit_base_options_warnings(enabled_opts)
 
-    def program_from_cross_file(self, prognames, silent=False):
-        cross_info = self.environment.cross_info
+    def _program_from_file(self, prognames, bins, silent):
         for p in prognames:
             if hasattr(p, 'held_object'):
                 p = p.held_object
@@ -2757,11 +2756,19 @@ external dependencies (including libraries) must go to "dependencies".''')
                 continue # Always points to a local (i.e. self generated) file.
             if not isinstance(p, str):
                 raise InterpreterException('Executable name must be a string')
-            prog = ExternalProgram.from_cross_info(cross_info, p)
+            prog = ExternalProgram.from_bin_list(bins, p)
             if prog.found():
                 return ExternalProgramHolder(prog)
         return None
 
+    def program_from_cross_file(self, prognames, silent=False):
+        bins = self.environment.cross_info.config['binaries']
+        return self._program_from_file(prognames, bins, silent)
+
+    def program_from_config_file(self, prognames, silent=False):
+        bins = self.environment.config_info.binaries
+        return self._program_from_file(prognames, bins, silent)
+
     def program_from_system(self, args, silent=False):
         # Search for scripts relative to current subdir.
         # Do not cache found programs because find_program('foobar')
@@ -2816,10 +2823,14 @@ external dependencies (including libraries) must go to "dependencies".''')
     def find_program_impl(self, args, native=False, required=True, silent=True):
         if not isinstance(args, list):
             args = [args]
+
         progobj = self.program_from_overrides(args, silent=silent)
-        if progobj is None and self.build.environment.is_cross_build():
-            if not native:
+        if progobj is None:
+            if self.build.environment.is_cross_build() and not native:
                 progobj = self.program_from_cross_file(args, silent=silent)
+            else:
+                progobj = self.program_from_config_file(args, silent=silent)
+
         if progobj is None:
             progobj = self.program_from_system(args, silent=silent)
         if required and (progobj is None or not progobj.found()):

mesonbuild/modules/python.py

@@ -480,7 +480,9 @@ class PythonModule(ExtensionModule):
         if len(args) > 1:
             raise InvalidArguments('find_installation takes zero or one positional argument.')
 
-        if args:
+        if 'python' in state.environment.config_info.binaries:
+            name_or_path = state.environment.config_info.binaries['python']
+        elif args:
             name_or_path = args[0]
             if not isinstance(name_or_path, str):
                 raise InvalidArguments('find_installation argument must be a string.')

mesonbuild/modules/python3.py

@@ -48,7 +48,10 @@ class Python3Module(ExtensionModule):
 
     @noKwargs
     def find_python(self, state, args, kwargs):
-        py3 = dependencies.ExternalProgram('python3', mesonlib.python_command, silent=True)
+        options = [state.environment.config_info.binaries.get('python3')]
+        if not options[0]:  # because this would be [None]
+            options = ['python3', mesonlib.python_command]
+        py3 = dependencies.ExternalProgram(*options, silent=True)
         return ModuleReturnValue(py3, [py3])
 
     @noKwargs

mesonbuild/modules/qt.py

@@ -18,7 +18,7 @@ from .. import build
 from ..mesonlib import MesonException, Popen_safe, extract_as_list, File
 from ..dependencies import Dependency, Qt4Dependency, Qt5Dependency
 import xml.etree.ElementTree as ET
-from . import ModuleReturnValue, get_include_args
+from . import ModuleReturnValue, get_include_args, ExtensionModule
 from ..interpreterbase import permittedKwargs, FeatureNewKwargs
 
 _QT_DEPS_LUT = {
@@ -27,10 +27,11 @@ _QT_DEPS_LUT = {
 }
 
 
-class QtBaseModule:
+class QtBaseModule(ExtensionModule):
     tools_detected = False
 
-    def __init__(self, qt_version=5):
+    def __init__(self, interpreter, qt_version=5):
+        ExtensionModule.__init__(self, interpreter)
         self.qt_version = qt_version
 
     def _detect_tools(self, env, method):
@@ -43,7 +44,7 @@ class QtBaseModule:
         kwargs = {'required': 'true', 'modules': 'Core', 'silent': 'true', 'method': method}
         qt = _QT_DEPS_LUT[self.qt_version](env, kwargs)
         # Get all tools and then make sure that they are the right version
-        self.moc, self.uic, self.rcc, self.lrelease = qt.compilers_detect()
+        self.moc, self.uic, self.rcc, self.lrelease = qt.compilers_detect(self.interpreter)
         # Moc, uic and rcc write their version strings to stderr.
         # Moc and rcc return a non-zero result when doing so.
         # What kind of an idiot thought that was a good idea?

mesonbuild/modules/qt4.py

@@ -14,14 +14,13 @@
 
 from .. import mlog
 from .qt import QtBaseModule
-from . import ExtensionModule
 
 
-class Qt4Module(ExtensionModule, QtBaseModule):
+class Qt4Module(QtBaseModule):
 
     def __init__(self, interpreter):
-        QtBaseModule.__init__(self, qt_version=4)
-        ExtensionModule.__init__(self, interpreter)
+        QtBaseModule.__init__(self, interpreter, qt_version=4)
+
 
 def initialize(*args, **kwargs):
     mlog.warning('rcc dependencies will not work properly until this upstream issue is fixed:',

mesonbuild/modules/qt5.py

@@ -14,14 +14,13 @@
 
 from .. import mlog
 from .qt import QtBaseModule
-from . import ExtensionModule
 
 
-class Qt5Module(ExtensionModule, QtBaseModule):
+class Qt5Module(QtBaseModule):
 
     def __init__(self, interpreter):
-        QtBaseModule.__init__(self, qt_version=5)
-        ExtensionModule.__init__(self, interpreter)
+        QtBaseModule.__init__(self, interpreter, qt_version=5)
+
 
 def initialize(*args, **kwargs):
     mlog.warning('rcc dependencies will not work reliably until this upstream issue is fixed:',

mesonbuild/modules/windows.py

@@ -49,8 +49,8 @@ class WindowsModule(ExtensionModule):
         if state.environment.is_cross_build():
             # If cross compiling see if windres has been specified in the
             # cross file before trying to find it another way.
-            cross_info = state.environment.cross_info
-            rescomp = ExternalProgram.from_cross_info(cross_info, 'windres')
+            bins = state.environment.cross_info.config['binaries']
+            rescomp = ExternalProgram.from_bin_list(bins, 'windres')
 
         if not rescomp or not rescomp.found():
             if 'WINDRES' in os.environ:
@@ -58,6 +58,13 @@ class WindowsModule(ExtensionModule):
                 # specifying an arch-specific windres.
                 rescomp = ExternalProgram('windres', command=os.environ.get('WINDRES'), silent=True)
 
+        if not rescomp or not rescomp.found():
+            # Take windres from the config file after the environment, which is
+            # in keeping with the expectations on unix-like OSes that
+            # environment variables trump config files.
+            bins = state.environment.config_info.binaries
+            rescomp = ExternalProgram.from_bin_list(bins, 'windres')
+
         if not rescomp or not rescomp.found():
             comp = self.detect_compiler(state.compilers)
             if comp.id == 'msvc' or comp.id == 'clang-cl':

mesonbuild/msetup.py

@@ -29,6 +29,10 @@ def add_arguments(parser):
     coredata.register_builtin_arguments(parser)
     parser.add_argument('--cross-file', default=None,
                         help='File describing cross compilation environment.')
+    parser.add_argument('--native-file',
+                        default=[],
+                        action='append',
+                        help='File containing overrides for native compilation environment.')
     parser.add_argument('-v', '--version', action='version',
                         version=coredata.version)
     parser.add_argument('--profile-self', action='store_true', dest='profile',

run_tests.py

@@ -73,6 +73,7 @@ def get_fake_options(prefix):
     opts.wrap_mode = None
     opts.prefix = prefix
     opts.cmd_line_options = {}
+    opts.native_file = []
     return opts
 
 def get_fake_env(sdir, bdir, prefix):

run_unittests.py

@@ -26,6 +26,7 @@ import sys
 import unittest
 import platform
 import pickle
+import functools
 from itertools import chain
 from unittest import mock
 from configparser import ConfigParser
@@ -41,7 +42,7 @@ import mesonbuild.modules.gnome
 from mesonbuild.interpreter import Interpreter, ObjectHolder
 from mesonbuild.mesonlib import (
     is_windows, is_osx, is_cygwin, is_dragonflybsd, is_openbsd, is_haiku,
-    windows_proof_rmtree, python_command, version_compare,
+    is_linux, windows_proof_rmtree, python_command, version_compare,
     BuildDirLock, Version
 )
 from mesonbuild.environment import detect_ninja
@@ -108,12 +109,41 @@ def skipIfNoPkgconfig(f):
 
     Note: Yes, we provide pkg-config even while running Windows CI
     '''
+    @functools.wraps(f)
     def wrapped(*args, **kwargs):
         if not is_ci() and shutil.which('pkg-config') is None:
             raise unittest.SkipTest('pkg-config not found')
         return f(*args, **kwargs)
     return wrapped
 
+def skip_if_not_language(lang):
+    def wrapper(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            try:
+                env = get_fake_env('', '', '')
+                f = getattr(env, 'detect_{}_compiler'.format(lang))
+                if lang in ['cs', 'vala', 'java', 'swift']:
+                    f()
+                else:
+                    f(False)
+            except EnvironmentException:
+                raise unittest.SkipTest('No {} compiler found.'.format(lang))
+            return func(*args, **kwargs)
+        return wrapped
+    return wrapper
+
+def skip_if_env_value(value):
+    def wrapper(func):
+        @functools.wraps(func)
+        def wrapped(*args, **kwargs):
+            if value in os.environ:
+                raise unittest.SkipTest(
+                    'Environment variable "{}" set, skipping.'.format(value))
+            return func(*args, **kwargs)
+        return wrapped
+    return wrapper
+
 class PatchModule:
     '''
     Fancy monkey-patching! Whee! Can't use mock.patch because it only
@@ -4446,6 +4476,273 @@ class RewriterTests(unittest.TestCase):
         self.assertEqual(s2, self.read_contents('sub2/meson.build'))
 
 
+class NativeFileTests(BasePlatformTests):
+
+    def setUp(self):
+        super().setUp()
+        self.testcase = os.path.join(self.unit_test_dir, '46 native file binary')
+        self.current_config = 0
+        self.current_wrapper = 0
+
+    def helper_create_native_file(self, values):
+        """Create a config file as a temporary file.
+
+        values should be a nested dictionary structure of {section: {key:
+        value}}
+        """
+        filename = os.path.join(self.builddir, 'generated{}.config'.format(self.current_config))
+        self.current_config += 1
+        with open(filename, 'wt') as f:
+            for section, entries in values.items():
+                f.write('[{}]\n'.format(section))
+                for k, v in entries.items():
+                    f.write("{}='{}'\n".format(k, v))
+        return filename
+
+    def helper_create_binary_wrapper(self, binary, **kwargs):
+        """Creates a wrapper around a binary that overrides specific values."""
+        filename = os.path.join(self.builddir, 'binary_wrapper{}.py'.format(self.current_wrapper))
+        self.current_wrapper += 1
+        if is_haiku():
+            chbang = '#!/bin/env python3'
+        else:
+            chbang = '#!/usr/bin/env python3'
+
+        with open(filename, 'wt') as f:
+            f.write(textwrap.dedent('''\
+                {}
+                import argparse
+                import subprocess
+                import sys
+
+                def main():
+                    parser = argparse.ArgumentParser()
+                '''.format(chbang)))
+            for name in kwargs:
+                f.write('    parser.add_argument("-{0}", "--{0}", action="store_true")\n'.format(name))
+            f.write('    args, extra_args = parser.parse_known_args()\n')
+            for name, value in kwargs.items():
+                f.write('    if args.{}:\n'.format(name))
+                f.write('        print("{}", file=sys.{})\n'.format(value, kwargs.get('outfile', 'stdout')))
+                f.write('        sys.exit(0)\n')
+            f.write(textwrap.dedent('''
+                    ret = subprocess.run(
+                        ["{}"] + extra_args,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        encoding='utf-8')
+                    print(ret.stdout)
+                    print(ret.stderr, file=sys.stderr)
+                    sys.exit(ret.returncode)
+
+                if __name__ == '__main__':
+                    main()
+                '''.format(binary)))
+
+        if not is_windows():
+            os.chmod(filename, 0o755)
+            return filename
+
+        # On windows we need yet another level of indirection, as cmd cannot
+        # invoke python files itself, so instead we generate a .bat file, which
+        # invokes our python wrapper
+        batfile = os.path.join(self.builddir, 'binary_wrapper{}.bat'.format(self.current_wrapper))
+        with open(batfile, 'wt') as f:
+            f.write('py -3 {} %*'.format(filename))
+        return batfile
+
+    def helper_for_compiler(self, lang, cb):
+        """Helper for generating tests for overriding compilers for langaugages
+        with more than one implementation, such as C, C++, ObjC, ObjC++, and D.
+        """
+        env = get_fake_env('', '', '')
+        getter = getattr(env, 'detect_{}_compiler'.format(lang))
+        if lang not in ['cs']:
+            getter = functools.partial(getter, False)
+        cc = getter()
+        binary, newid = cb(cc)
+        env.config_info.binaries = {lang: binary}
+        compiler = getter()
+        self.assertEqual(compiler.id, newid)
+
+    def test_multiple_native_files_override(self):
+        wrapper = self.helper_create_binary_wrapper('bash', version='foo')
+        config = self.helper_create_native_file({'binaries': {'bash': wrapper}})
+        wrapper = self.helper_create_binary_wrapper('bash', version='12345')
+        config2 = self.helper_create_native_file({'binaries': {'bash': wrapper}})
+        self.init(self.testcase, extra_args=[
+            '--native-file', config, '--native-file', config2,
+            '-Dcase=find_program'])
+
+    def test_multiple_native_files(self):
+        wrapper = self.helper_create_binary_wrapper('bash', version='12345')
+        config = self.helper_create_native_file({'binaries': {'bash': wrapper}})
+        wrapper = self.helper_create_binary_wrapper('python')
+        config2 = self.helper_create_native_file({'binaries': {'python': wrapper}})
+        self.init(self.testcase, extra_args=[
+            '--native-file', config, '--native-file', config2,
+            '-Dcase=find_program'])
+
+    def _simple_test(self, case, binary):
+        wrapper = self.helper_create_binary_wrapper(binary, version='12345')
+        config = self.helper_create_native_file({'binaries': {binary: wrapper}})
+        self.init(self.testcase, extra_args=['--native-file', config, '-Dcase={}'.format(case)])
+
+    def test_find_program(self):
+        self._simple_test('find_program', 'bash')
+
+    def test_config_tool_dep(self):
+        # Do the skip at this level to avoid screwing up the cache
+        if not shutil.which('llvm-config'):
+            raise unittest.SkipTest('No llvm-installed, cannot test')
+        self._simple_test('config_dep', 'llvm-config')
+
+    def test_python3_module(self):
+        self._simple_test('python3', 'python3')
+
+    def test_python_module(self):
+        if is_windows():
+            # Bat adds extra crap to stdout, so the version check logic in the
+            # python module breaks. This is fine on other OSes because they
+            # don't need the extra indirection.
+            raise unittest.SkipTest('bat indirection breaks internal sanity checks.')
+        self._simple_test('python', 'python')
+
+    @unittest.skipIf(is_windows(), 'Setting up multiple compilers on windows is hard')
+    @skip_if_env_value('CC')
+    def test_c_compiler(self):
+        def cb(comp):
+            if comp.id == 'gcc':
+                if not shutil.which('clang'):
+                    raise unittest.SkipTest('Only one compiler found, cannot test.')
+                return 'clang', 'clang'
+            if not shutil.which('gcc'):
+                raise unittest.SkipTest('Only one compiler found, cannot test.')
+            return 'gcc', 'gcc'
+        self.helper_for_compiler('c', cb)
+
+    @unittest.skipIf(is_windows(), 'Setting up multiple compilers on windows is hard')
+    @skip_if_env_value('CXX')
+    def test_cpp_compiler(self):
+        def cb(comp):
+            if comp.id == 'gcc':
+                if not shutil.which('clang++'):
+                    raise unittest.SkipTest('Only one compiler found, cannot test.')
+                return 'clang++', 'clang'
+            if not shutil.which('g++'):
+                raise unittest.SkipTest('Only one compiler found, cannot test.')
+            return 'g++', 'gcc'
+        self.helper_for_compiler('cpp', cb)
+
+    @skip_if_not_language('objc')
+    @skip_if_env_value('OBJC')
+    def test_objc_compiler(self):
+        def cb(comp):
+            if comp.id == 'gcc':
+                if not shutil.which('clang'):
+                    raise unittest.SkipTest('Only one compiler found, cannot test.')
+                return 'clang', 'clang'
+            if not shutil.which('gcc'):
+                raise unittest.SkipTest('Only one compiler found, cannot test.')
+            return 'gcc', 'gcc'
+        self.helper_for_compiler('objc', cb)
+
+    @skip_if_not_language('objcpp')
+    @skip_if_env_value('OBJCXX')
+    def test_objcpp_compiler(self):
+        def cb(comp):
+            if comp.id == 'gcc':
+                if not shutil.which('clang++'):
+                    raise unittest.SkipTest('Only one compiler found, cannot test.')
+                return 'clang++', 'clang'
+            if not shutil.which('g++'):
+                raise unittest.SkipTest('Only one compiler found, cannot test.')
+            return 'g++', 'gcc'
+        self.helper_for_compiler('objcpp', cb)
+
+    @skip_if_not_language('d')
+    @skip_if_env_value('DC')
+    def test_d_compiler(self):
+        def cb(comp):
+            if comp.id == 'dmd':
+                if shutil.which('ldc'):
+                    return 'ldc', 'ldc'
+                elif shutil.which('gdc'):
+                    return 'gdc', 'gdc'
+                else:
+                    raise unittest.SkipTest('No alternative dlang compiler found.')
+            return 'dmd', 'dmd'
+        self.helper_for_compiler('d', cb)
+
+    @skip_if_not_language('cs')
+    @skip_if_env_value('CSC')
+    def test_cs_compiler(self):
+        def cb(comp):
+            if comp.id == 'csc':
+                if not shutil.which('mcs'):
+                    raise unittest.SkipTest('No alternate C# implementation.')
+                return 'mcs', 'mcs'
+            if not shutil.which('csc'):
+                raise unittest.SkipTest('No alternate C# implementation.')
+            return 'csc', 'csc'
+        self.helper_for_compiler('cs', cb)
+
+    @skip_if_not_language('fortran')
+    @skip_if_env_value('FC')
+    def test_fortran_compiler(self):
+        def cb(comp):
+            if comp.id == 'gcc':
+                if shutil.which('ifort'):
+                    return 'ifort', 'intel'
+                # XXX: there are several other fortran compilers meson
+                # supports, but I don't have any of them to test with
+                raise unittest.SkipTest('No alternate Fortran implementation.')
+            if not shutil.which('gfortran'):
+                raise unittest.SkipTest('No alternate C# implementation.')
+            return 'gfortran', 'gcc'
+        self.helper_for_compiler('fortran', cb)
+
+    def _single_implementation_compiler(self, lang, binary, version_str, version):
+        """Helper for languages with a single (supported) implementation.
+
+        Builds a wrapper around the compiler to override the version.
+        """
+        wrapper = self.helper_create_binary_wrapper(binary, version=version_str)
+        env = get_fake_env('', '', '')
+        getter = getattr(env, 'detect_{}_compiler'.format(lang))
+        if lang in ['rust']:
+            getter = functools.partial(getter, False)
+        env.config_info.binaries = {lang: wrapper}
+        compiler = getter()
+        self.assertEqual(compiler.version, version)
+
+    @skip_if_not_language('vala')
+    @skip_if_env_value('VALAC')
+    def test_vala_compiler(self):
+        self._single_implementation_compiler(
+            'vala', 'valac', 'Vala 1.2345', '1.2345')
+
+    @skip_if_not_language('rust')
+    @skip_if_env_value('RUSTC')
+    def test_rust_compiler(self):
+        self._single_implementation_compiler(
+            'rust', 'rustc', 'rustc 1.2345', '1.2345')
+
+    @skip_if_not_language('java')
+    def test_java_compiler(self):
+        self._single_implementation_compiler(
+            'java', 'javac', 'javac 9.99.77', '9.99.77')
+
+    @skip_if_not_language('swift')
+    def test_swift_compiler(self):
+        wrapper = self.helper_create_binary_wrapper(
+            'swiftc', version='Swift 1.2345', outfile='stderr')
+        env = get_fake_env('', '', '')
+        env.config_info.binaries = {'swift': wrapper}
+        compiler = env.detect_swift_compiler()
+        self.assertEqual(compiler.version, '1.2345')
+
+
 def unset_envs():
     # For unit tests we must fully control all command lines
     # so that there are no unexpected changes coming from the
@@ -4463,7 +4760,8 @@ def should_run_cross_mingw_tests():
 
 def main():
     unset_envs()
-    cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests', 'PythonTests']
+    cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests',
+             'PythonTests', 'NativeFileTests']
     if not is_windows():
         cases += ['LinuxlikeTests']
         if should_run_cross_arm_tests():

test cases/unit/46 native file binary/meson.build

@@ -0,0 +1,21 @@
+project('test project')
+
+case = get_option('case')
+
+if case == 'find_program'
+  prog = find_program('bash')
+  result = run_command(prog, ['--version'])
+  assert(result.stdout().strip().endswith('12345'), 'Didn\'t load bash from config file')
+elif case == 'config_dep'
+  add_languages('cpp')
+  dep = dependency('llvm')
+  assert(dep.get_configtool_variable('version').endswith('12345'), 'Didn\'t load llvm from config file')
+elif case == 'python3'
+  prog = import('python3').find_python()
+  result = run_command(prog, ['--version'])
+  assert(result.stdout().strip().endswith('12345'), 'Didn\'t load python3 from config file')
+elif case == 'python'
+  prog = import('python').find_installation()
+  result = run_command(prog, ['--version'])
+  assert(result.stdout().strip().endswith('12345'), 'Didn\'t load python from config file')
+endif

test cases/unit/46 native file binary/meson_options.txt

@@ -0,0 +1,5 @@
+option(
+  'case',
+  type : 'combo',
+  choices : ['find_program', 'config_dep', 'python3', 'python']
+)