Merge pull request #8159 from dcbaker/submit/all-env-variables-up-front

Read and store all environment variables up front

mesonbuild/backend/backends.py

@@ -29,7 +29,7 @@ from .. import build
 from .. import dependencies
 from .. import mesonlib
 from .. import mlog
-from ..compilers import languages_using_ldflags
+from ..compilers import LANGUAGES_USING_LDFLAGS
 from ..mesonlib import (
     File, MachineChoice, MesonException, OptionType, OrderedSet, OptionOverrideProxy,
     classify_unity_sources, unholder, OptionKey
@@ -480,7 +480,7 @@ class Backend:
     def get_external_rpath_dirs(self, target):
         dirs = set()
         args = []
-        for lang in languages_using_ldflags:
+        for lang in LANGUAGES_USING_LDFLAGS:
             try:
                 args.extend(self.environment.coredata.get_external_link_args(target.for_machine, lang))
             except Exception:

mesonbuild/cmake/executor.py

@@ -24,7 +24,6 @@ import os
 
 from .. import mlog
 from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey
-from ..envconfig import get_env_var
 
 if T.TYPE_CHECKING:
     from ..environment import Environment
@@ -63,23 +62,6 @@ class CMakeExecutor:
             return
 
         self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value
-        env_pref_path_raw = get_env_var(
-            self.for_machine,
-            self.environment.is_cross_build(),
-            'CMAKE_PREFIX_PATH')
-        if env_pref_path_raw is not None:
-            env_pref_path = []  # type: T.List[str]
-            if is_windows():
-                # Cannot split on ':' on Windows because its in the drive letter
-                env_pref_path = env_pref_path_raw.split(os.pathsep)
-            else:
-                # https://github.com/mesonbuild/meson/issues/7294
-                env_pref_path = re.split(r':|;', env_pref_path_raw)
-            env_pref_path = [x for x in env_pref_path if x]  # Filter out empty strings
-            if not self.prefix_paths:
-                self.prefix_paths = []
-            self.prefix_paths += env_pref_path
-
         if self.prefix_paths:
             self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))]
 

mesonbuild/compilers/__init__.py

@@ -127,7 +127,7 @@ from .compilers import (
     is_library,
     is_known_suffix,
     lang_suffixes,
-    languages_using_ldflags,
+    LANGUAGES_USING_LDFLAGS,
     sort_clink,
 )
 from .c import (

mesonbuild/compilers/c.py

@@ -17,7 +17,6 @@ import typing as T
 
 from .. import coredata
 from ..mesonlib import MachineChoice, MesonException, mlog, version_compare, OptionKey
-from ..linkers import LinkerEnvVarsMixin
 from .c_function_attributes import C_FUNC_ATTRIBUTES
 from .mixins.clike import CLikeCompiler
 from .mixins.ccrx import CcrxCompiler
@@ -195,7 +194,7 @@ class AppleClangCCompiler(ClangCCompiler):
     _C2X_VERSION = '>=11.0.0'
 
 
-class EmscriptenCCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCCompiler):
+class EmscriptenCCompiler(EmscriptenMixin, ClangCCompiler):
     def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool,
                  info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None,
                  linker: T.Optional['DynamicLinker'] = None,

mesonbuild/compilers/compilers.py

@@ -22,13 +22,9 @@ from functools import lru_cache
 from .. import coredata
 from .. import mlog
 from .. import mesonlib
-from ..linkers import LinkerEnvVarsMixin
 from ..mesonlib import (
     EnvironmentException, MachineChoice, MesonException,
-    Popen_safe, split_args, LibType, TemporaryDirectoryWinProof, OptionKey,
-)
-from ..envconfig import (
-    get_env_var
+    Popen_safe, LibType, TemporaryDirectoryWinProof, OptionKey,
 )
 
 from ..arglist import CompilerArgs
@@ -85,24 +81,28 @@ clink_suffixes += ('h', 'll', 's')
 all_suffixes = set(itertools.chain(*lang_suffixes.values(), clink_suffixes))  # type: T.Set[str]
 
 # Languages that should use LDFLAGS arguments when linking.
-languages_using_ldflags = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'}  # type: T.Set[str]
+LANGUAGES_USING_LDFLAGS = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'}  # type: T.Set[str]
 # Languages that should use CPPFLAGS arguments when linking.
-languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'}  # type: T.Set[str]
+LANGUAGES_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcpp'}  # type: T.Set[str]
 soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$')
 
 # Environment variables that each lang uses.
-cflags_mapping = {'c': 'CFLAGS',
-                  'cpp': 'CXXFLAGS',
-                  'cuda': 'CUFLAGS',
-                  'objc': 'OBJCFLAGS',
-                  'objcpp': 'OBJCXXFLAGS',
-                  'fortran': 'FFLAGS',
-                  'd': 'DFLAGS',
-                  'vala': 'VALAFLAGS',
-                  'rust': 'RUSTFLAGS'}  # type: T.Dict[str, str]
-
-cexe_mapping = {'c': 'CC',
-                'cpp': 'CXX'}
+CFLAGS_MAPPING: T.Mapping[str, str] = {
+    'c': 'CFLAGS',
+    'cpp': 'CXXFLAGS',
+    'cuda': 'CUFLAGS',
+    'objc': 'OBJCFLAGS',
+    'objcpp': 'OBJCXXFLAGS',
+    'fortran': 'FFLAGS',
+    'd': 'DFLAGS',
+    'vala': 'VALAFLAGS',
+    'rust': 'RUSTFLAGS',
+}
+
+CEXE_MAPPING: T.Mapping = {
+    'c': 'CC',
+    'cpp': 'CXX',
+}
 
 # All these are only for C-linkable languages; see `clink_langs` above.
 
@@ -587,11 +587,6 @@ class Compiler(metaclass=abc.ABCMeta):
         """
         return []
 
-    def get_linker_args_from_envvars(self,
-                                     for_machine: MachineChoice,
-                                     is_cross: bool) -> T.List[str]:
-        return self.linker.get_args_from_envvars(for_machine, is_cross)
-
     def get_options(self) -> 'KeyedOptionDictType':
         return {}
 
@@ -1199,68 +1194,32 @@ class Compiler(metaclass=abc.ABCMeta):
     def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]:
         raise EnvironmentException('{} does not know how to do prelinking.'.format(self.id))
 
-def get_args_from_envvars(lang: str,
-                          for_machine: MachineChoice,
-                          is_cross: bool,
-                          use_linker_args: bool) -> T.Tuple[T.List[str], T.List[str]]:
-    """
-    Returns a tuple of (compile_flags, link_flags) for the specified language
-    from the inherited environment
-    """
-    if lang not in cflags_mapping:
-        return [], []
-
-    compile_flags = []  # type: T.List[str]
-    link_flags = []     # type: T.List[str]
-
-    env_compile_flags = get_env_var(for_machine, is_cross, cflags_mapping[lang])
-    if env_compile_flags is not None:
-        compile_flags += split_args(env_compile_flags)
-
-    # Link flags (same for all languages)
-    if lang in languages_using_ldflags:
-        link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross)
-    if use_linker_args:
-        # When the compiler is used as a wrapper around the linker (such as
-        # with GCC and Clang), the compile flags can be needed while linking
-        # too. This is also what Autotools does. However, we don't want to do
-        # this when the linker is stand-alone such as with MSVC C/C++, etc.
-        link_flags = compile_flags + link_flags
-
-    # Pre-processor flags for certain languages
-    if lang in languages_using_cppflags:
-        env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS')
-        if env_preproc_flags is not None:
-            compile_flags += split_args(env_preproc_flags)
-
-    return compile_flags, link_flags
-
 
 def get_global_options(lang: str,
                        comp: T.Type[Compiler],
                        for_machine: MachineChoice,
-                       is_cross: bool) -> 'KeyedOptionDictType':
+                       env: 'Environment') -> 'KeyedOptionDictType':
     """Retreive options that apply to all compilers for a given language."""
     description = 'Extra arguments passed to the {}'.format(lang)
     argkey = OptionKey('args', lang=lang, machine=for_machine)
     largkey = argkey.evolve('link_args')
+
+    # We shouldn't need listify here, but until we have a separate
+    # linker-driver representation and can have that do the combine we have to
+    # do it htis way.
+    compile_args = mesonlib.listify(env.options.get(argkey, []))
+    link_args = mesonlib.listify(env.options.get(largkey, []))
+
+    if comp.INVOKES_LINKER:
+        link_args = compile_args + link_args
+
     opts: 'KeyedOptionDictType' = {
         argkey: coredata.UserArrayOption(
             description + ' compiler',
-            [], split_args=True, user_input=True, allow_dups=True),
+            compile_args, split_args=True, user_input=True, allow_dups=True),
         largkey: coredata.UserArrayOption(
             description + ' linker',
-            [], split_args=True, user_input=True, allow_dups=True),
+            link_args, split_args=True, user_input=True, allow_dups=True),
     }
 
-    # Get from env vars.
-    compile_args, link_args = get_args_from_envvars(
-        lang,
-        for_machine,
-        is_cross,
-        comp.INVOKES_LINKER)
-
-    opts[argkey].set_value(compile_args)
-    opts[largkey].set_value(link_args)
-
     return opts

mesonbuild/compilers/cpp.py

@@ -21,7 +21,6 @@ from .. import coredata
 from .. import mlog
 from ..mesonlib import MesonException, MachineChoice, version_compare, OptionKey
 
-from ..linkers import LinkerEnvVarsMixin
 from .compilers import (
     gnu_winlibs,
     msvc_winlibs,
@@ -256,7 +255,7 @@ class AppleClangCPPCompiler(ClangCPPCompiler):
         return ['-lc++']
 
 
-class EmscriptenCPPCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCPPCompiler):
+class EmscriptenCPPCompiler(EmscriptenMixin, ClangCPPCompiler):
     def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool,
                  info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None,
                  linker: T.Optional['DynamicLinker'] = None,

mesonbuild/coredata.py

@@ -759,7 +759,7 @@ class CoreData:
                       for_machine: MachineChoice, env: 'Environment') -> None:
         """Add global language arguments that are needed before compiler/linker detection."""
         from .compilers import compilers
-        options = compilers.get_global_options(lang, comp, for_machine, env.is_cross_build())
+        options = compilers.get_global_options(lang, comp, for_machine, env)
         self.add_compiler_options(options, lang, for_machine, env)
 
     def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None:

mesonbuild/dependencies/base.py

@@ -32,7 +32,6 @@ from pathlib import Path, PurePath
 from .. import mlog
 from .. import mesonlib
 from ..compilers import clib_langs
-from ..envconfig import get_env_var
 from ..environment import Environment, MachineInfo
 from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException, CMakeToolchain, CMakeExecScope, check_cmake_args
 from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine
@@ -317,7 +316,7 @@ class HasNativeKwarg:
         return MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST
 
 class ExternalDependency(Dependency, HasNativeKwarg):
-    def __init__(self, type_name, environment, kwargs, language: T.Optional[str] = None):
+    def __init__(self, type_name, environment: Environment, kwargs, language: T.Optional[str] = None):
         Dependency.__init__(self, type_name, kwargs)
         self.env = environment
         self.name = type_name # default
@@ -784,14 +783,7 @@ class PkgConfigDependency(ExternalDependency):
         #
         # Only prefix_libpaths are reordered here because there should not be
         # too many system_libpaths to cause library version issues.
-        pkg_config_path = get_env_var(
-            self.for_machine,
-            self.env.is_cross_build(),
-            'PKG_CONFIG_PATH')
-        if pkg_config_path:
-            pkg_config_path = pkg_config_path.split(os.pathsep)
-        else:
-            pkg_config_path = []
+        pkg_config_path: T.List[str] = self.env.coredata.options[OptionKey('pkg_config_path', machine=self.for_machine)].value
         pkg_config_path = self._convert_mingw_paths(pkg_config_path)
         prefix_libpaths = sort_libpaths(prefix_libpaths, pkg_config_path)
         system_libpaths = OrderedSet()

mesonbuild/dependencies/boost.py

@@ -20,12 +20,14 @@ from pathlib import Path
 
 from .. import mlog
 from .. import mesonlib
-from ..envconfig import get_env_var
 from ..environment import Environment
 
 from .base import DependencyException, ExternalDependency, PkgConfigDependency
 from .misc import threads_factory
 
+if T.TYPE_CHECKING:
+    from ..environment import Properties
+
 # On windows 3 directory layouts are supported:
 # * The default layout (versioned) installed:
 #   - $BOOST_ROOT/include/boost-x_x/boost/*.hpp
@@ -372,18 +374,11 @@ class BoostDependency(ExternalDependency):
 
         # First, look for paths specified in a machine file
         props = self.env.properties[self.for_machine]
-        boost_property_env = [props.get('boost_includedir'), props.get('boost_librarydir'), props.get('boost_root')]
-        if any(boost_property_env):
+        if any(x in self.env.properties[self.for_machine] for x in
+               ['boost_includedir', 'boost_librarydir', 'boost_root']):
             self.detect_boost_machine_file(props)
             return
 
-        # Next, look for paths in the environment
-        boost_manual_env_list = ['BOOST_INCLUDEDIR', 'BOOST_LIBRARYDIR', 'BOOST_ROOT', 'BOOSTROOT']
-        boost_manual_env = [get_env_var(self.for_machine, self.env.is_cross_build, x) for x in boost_manual_env_list]
-        if any(boost_manual_env):
-            self.detect_boost_env()
-            return
-
         # Finally, look for paths from .pc files and from searching the filesystem
         self.detect_roots()
 
@@ -405,13 +400,20 @@ class BoostDependency(ExternalDependency):
                 self.boost_root = j
                 break
 
-    def detect_boost_machine_file(self, props: T.Dict[str, str]) -> None:
+    def detect_boost_machine_file(self, props: 'Properties') -> None:
+        """Detect boost with values in the machine file or environment.
+
+        The machine file values are defaulted to the environment values.
+        """
+        # XXX: if we had a TypedDict we woudn't need this
         incdir = props.get('boost_includedir')
+        assert incdir is None or isinstance(incdir, str)
         libdir = props.get('boost_librarydir')
+        assert libdir is None or isinstance(libdir, str)
 
         if incdir and libdir:
-            inc_dir = Path(props['boost_includedir'])
-            lib_dir = Path(props['boost_librarydir'])
+            inc_dir = Path(incdir)
+            lib_dir = Path(libdir)
 
             if not inc_dir.is_absolute() or not lib_dir.is_absolute():
                 raise DependencyException('Paths given for boost_includedir and boost_librarydir in machine file must be absolute')
@@ -436,43 +438,6 @@ class BoostDependency(ExternalDependency):
 
         self.check_and_set_roots(paths)
 
-    def detect_boost_env(self) -> None:
-        boost_includedir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_INCLUDEDIR')
-        boost_librarydir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_LIBRARYDIR')
-
-        boost_manual_env = [boost_includedir, boost_librarydir]
-        if all(boost_manual_env):
-            inc_dir = Path(boost_includedir)
-            lib_dir = Path(boost_librarydir)
-
-            if not inc_dir.is_absolute() or not lib_dir.is_absolute():
-                raise DependencyException('Paths given in BOOST_INCLUDEDIR and BOOST_LIBRARYDIR must be absolute')
-
-            mlog.debug('Trying to find boost with:')
-            mlog.debug('  - BOOST_INCLUDEDIR = {}'.format(inc_dir))
-            mlog.debug('  - BOOST_LIBRARYDIR = {}'.format(lib_dir))
-
-            return self.detect_split_root(inc_dir, lib_dir)
-
-        elif any(boost_manual_env):
-            raise DependencyException('Both BOOST_INCLUDEDIR *and* BOOST_LIBRARYDIR have to be set (one is not enough). Ignoring.')
-
-        boost_root = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_ROOT')
-        boostroot = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOSTROOT')
-
-        # It shouldn't be possible to get here without something in BOOST_ROOT or BOOSTROOT
-        assert(boost_root or boostroot)
-
-        for path, name in [(boost_root, 'BOOST_ROOT'), (boostroot, 'BOOSTROOT')]:
-            if path:
-                raw_paths = path.split(os.pathsep)
-                paths = [Path(x) for x in raw_paths]
-                if paths and any([not x.is_absolute() for x in paths]):
-                    raise DependencyException('Paths in {} must be absolute'.format(name))
-                break
-
-        self.check_and_set_roots(paths)
-
     def run_check(self, inc_dirs: T.List[BoostIncludeDir], lib_dirs: T.List[Path]) -> bool:
         mlog.debug('  - potential library dirs: {}'.format([x.as_posix() for x in lib_dirs]))
         mlog.debug('  - potential include dirs: {}'.format([x.path.as_posix() for x in inc_dirs]))

mesonbuild/dependencies/hdf5.py

@@ -30,8 +30,8 @@ import typing as T
 
 if T.TYPE_CHECKING:
     from .base import Dependency
-    from ..envconfig import MachineChoice
     from ..environment import Environment
+    from ..mesonlib import MachineChoice
 
 
 class HDF5PkgConfigDependency(PkgConfigDependency):

mesonbuild/dependencies/mpi.py

@@ -106,7 +106,7 @@ class _MPIConfigToolDependency(ConfigToolDependency):
         Drop -O2 and everything that is not needed.
         """
         result = []
-        multi_args = ('-I', )
+        multi_args: T.Tuple[str, ...] = ('-I', )
         if self.language == 'fortran':
             fc = self.env.coredata.compilers[self.for_machine]['fortran']
             multi_args += fc.get_module_incdir_args()

mesonbuild/envconfig.py

@@ -12,17 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os, subprocess
+import subprocess
 import typing as T
 from enum import Enum
 
 from . import mesonlib
-from .mesonlib import EnvironmentException, MachineChoice, PerMachine, split_args
+from .mesonlib import EnvironmentException
 from . import mlog
 from pathlib import Path
 
-_T = T.TypeVar('_T')
-
 
 # These classes contains all the data pulled from configuration files (native
 # and cross file currently), and also assists with the reading environment
@@ -85,46 +83,54 @@ CPU_FAMILES_64_BIT = [
     'x86_64',
 ]
 
+# Map from language identifiers to environment variables.
+ENV_VAR_PROG_MAP: T.Mapping[str, str] = {
+    # Compilers
+    'c': 'CC',
+    'cpp': 'CXX',
+    'cs': 'CSC',
+    'd': 'DC',
+    'fortran': 'FC',
+    'objc': 'OBJC',
+    'objcpp': 'OBJCXX',
+    'rust': 'RUSTC',
+    'vala': 'VALAC',
+
+    # Linkers
+    'c_ld': 'CC_LD',
+    'cpp_ld': 'CXX_LD',
+    'd_ld': 'DC_LD',
+    'fortran_ld': 'FC_LD',
+    'objc_ld': 'OBJC_LD',
+    'objcpp_ld': 'OBJCXX_LD',
+    'rust_ld': 'RUSTC_LD',
+
+    # Binutils
+    'strip': 'STRIP',
+    'ar': 'AR',
+    'windres': 'WINDRES',
+
+    # Other tools
+    'cmake': 'CMAKE',
+    'qmake': 'QMAKE',
+    'pkgconfig': 'PKG_CONFIG',
+    'make': 'MAKE',
+}
+
+# Deprecated environment variables mapped from the new variable to the old one
+# Deprecated in 0.54.0
+DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = {
+    'd_ld': 'D_LD',
+    'fortran_ld': 'F_LD',
+    'rust_ld': 'RUST_LD',
+    'objcpp_ld': 'OBJCPP_LD',
+}
+
 class CMakeSkipCompilerTest(Enum):
     ALWAYS = 'always'
     NEVER = 'never'
     DEP_ONLY = 'dep_only'
 
-
-def get_env_var_pair(for_machine: MachineChoice,
-                     is_cross: bool,
-                     var_name: str) -> T.Optional[T.Tuple[str, str]]:
-    """
-    Returns the exact env var and the value.
-    """
-    candidates = PerMachine(
-        # The prefixed build version takes priority, but if we are native
-        # compiling we fall back on the unprefixed host version. This
-        # allows native builds to never need to worry about the 'BUILD_*'
-        # ones.
-        ([var_name + '_FOR_BUILD'] if is_cross else [var_name]),
-        # Always just the unprefixed host verions
-        [var_name]
-    )[for_machine]
-    for var in candidates:
-        value = os.environ.get(var)
-        if value is not None:
-            break
-    else:
-        formatted = ', '.join(['{!r}'.format(var) for var in candidates])
-        mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted))
-        return None
-    mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value))
-    return var, value
-
-def get_env_var(for_machine: MachineChoice,
-                is_cross: bool,
-                var_name: str) -> T.Optional[str]:
-    ret = get_env_var_pair(for_machine, is_cross, var_name)
-    if ret is None:
-        return None
-    return ret[1]
-
 class Properties:
     def __init__(
             self,
@@ -351,60 +357,18 @@ class MachineInfo:
         return self.is_windows() or self.is_cygwin()
 
 class BinaryTable:
+
     def __init__(
             self,
             binaries: T.Optional[T.Dict[str, T.Union[str, T.List[str]]]] = None,
     ):
-        self.binaries = binaries or {}  # type: T.Dict[str, T.Union[str, T.List[str]]]
-        for name, command in self.binaries.items():
-            if not isinstance(command, (list, str)):
-                # TODO generalize message
-                raise mesonlib.MesonException(
-                    'Invalid type {!r} for binary {!r} in cross file'
-                    ''.format(command, name))
-
-    # Map from language identifiers to environment variables.
-    evarMap = {
-        # Compilers
-        'c': 'CC',
-        'cpp': 'CXX',
-        'cs': 'CSC',
-        'd': 'DC',
-        'fortran': 'FC',
-        'objc': 'OBJC',
-        'objcpp': 'OBJCXX',
-        'rust': 'RUSTC',
-        'vala': 'VALAC',
-
-        # Linkers
-        'c_ld': 'CC_LD',
-        'cpp_ld': 'CXX_LD',
-        'd_ld': 'DC_LD',
-        'fortran_ld': 'FC_LD',
-        'objc_ld': 'OBJC_LD',
-        'objcpp_ld': 'OBJCXX_LD',
-        'rust_ld': 'RUSTC_LD',
-
-        # Binutils
-        'strip': 'STRIP',
-        'ar': 'AR',
-        'windres': 'WINDRES',
-
-        # Other tools
-        'cmake': 'CMAKE',
-        'qmake': 'QMAKE',
-        'pkgconfig': 'PKG_CONFIG',
-        'make': 'MAKE',
-    }  # type: T.Dict[str, str]
-
-    # Deprecated environment variables mapped from the new variable to the old one
-    # Deprecated in 0.54.0
-    DEPRECATION_MAP = {
-        'DC_LD': 'D_LD',
-        'FC_LD': 'F_LD',
-        'RUSTC_LD': 'RUST_LD',
-        'OBJCXX_LD': 'OBJCPP_LD',
-    }  # type: T.Dict[str, str]
+        self.binaries: T.Dict[str, T.List[str]] = {}
+        if binaries:
+            for name, command in binaries.items():
+                if not isinstance(command, (list, str)):
+                    raise mesonlib.MesonException(
+                        f'Invalid type {command!r} for entry {name!r} in cross file')
+                self.binaries[name] = mesonlib.listify(command)
 
     @staticmethod
     def detect_ccache() -> T.List[str]:
@@ -426,42 +390,17 @@ class BinaryTable:
         # Return value has to be a list of compiler 'choices'
         return compiler, ccache
 
-    def lookup_entry(self,
-                     for_machine: MachineChoice,
-                     is_cross: bool,
-                     name: str) -> T.Optional[T.List[str]]:
+    def lookup_entry(self, name: str) -> T.Optional[T.List[str]]:
         """Lookup binary in cross/native file and fallback to environment.
 
         Returns command with args as list if found, Returns `None` if nothing is
         found.
         """
-        # Try explicit map, don't fall back on env var
-        # Try explict map, then env vars
-        for _ in [()]: # a trick to get `break`
-            raw_command = self.binaries.get(name)
-            if raw_command is not None:
-                command = mesonlib.stringlistify(raw_command)
-                break # found
-            evar = self.evarMap.get(name)
-            if evar is not None:
-                raw_command = get_env_var(for_machine, is_cross, evar)
-                if raw_command is None:
-                    deprecated = self.DEPRECATION_MAP.get(evar)
-                    if deprecated is not None:
-                        raw_command = get_env_var(for_machine, is_cross, deprecated)
-                        if raw_command is not None:
-                            mlog.deprecation(
-                                'The', deprecated, 'environment variable is deprecated in favor of',
-                                evar, once=True)
-                if raw_command is not None:
-                    command = split_args(raw_command)
-                    break # found
-            command = None
-
-
-        # Do not return empty or blank string entries
-        if command is not None and (len(command) == 0 or len(command[0].strip()) == 0):
-            command = None
+        command = self.binaries.get(name)
+        if not command:
+            return None
+        elif not command[0].strip():
+            return None
         return command
 
 class CMakeVariables:

mesonbuild/environment.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2016 The Meson development team
+# Copyright 2012-2020 The Meson development team
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import os, platform, re, sys, shutil, subprocess
 import tempfile
 import shlex
@@ -22,15 +23,13 @@ from . import coredata
 from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, CompCertLinker, C2000Linker, IntelVisualStudioLinker, AIXArLinker
 from . import mesonlib
 from .mesonlib import (
-    MesonException, EnvironmentException, MachineChoice, Popen_safe,
+    MesonException, EnvironmentException, MachineChoice, Popen_safe, PerMachine,
     PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey
 )
 from . import mlog
 
 from .envconfig import (
-    BinaryTable, MachineInfo,
-    Properties, known_cpu_families, get_env_var_pair,
-    CMakeVariables,
+    BinaryTable, MachineInfo, Properties, known_cpu_families, CMakeVariables,
 )
 from . import compilers
 from .compilers import (
@@ -128,6 +127,7 @@ from .compilers import (
     VisualStudioCCompiler,
     VisualStudioCPPCompiler,
 )
+from mesonbuild import envconfig
 
 if T.TYPE_CHECKING:
     from configparser import ConfigParser
@@ -141,6 +141,32 @@ CompilersDict = T.Dict[str, Compiler]
 if T.TYPE_CHECKING:
     import argparse
 
+
+def _get_env_var(for_machine: MachineChoice, is_cross: bool, var_name: str) -> T.Optional[str]:
+    """
+    Returns the exact env var and the value.
+    """
+    candidates = PerMachine(
+        # The prefixed build version takes priority, but if we are native
+        # compiling we fall back on the unprefixed host version. This
+        # allows native builds to never need to worry about the 'BUILD_*'
+        # ones.
+        ([var_name + '_FOR_BUILD'] if is_cross else [var_name]),
+        # Always just the unprefixed host verions
+        [var_name]
+    )[for_machine]
+    for var in candidates:
+        value = os.environ.get(var)
+        if value is not None:
+            break
+    else:
+        formatted = ', '.join(['{!r}'.format(var) for var in candidates])
+        mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted))
+        return None
+    mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value))
+    return value
+
+
 def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False):
     gcovr_exe = 'gcovr'
     try:
@@ -579,7 +605,7 @@ class Environment:
         # Stores machine infos, the only *three* machine one because we have a
         # target machine info on for the user (Meson never cares about the
         # target machine.)
-        machines = PerThreeMachineDefaultable()  # type: PerMachineDefaultable[MachineInfo]
+        machines: PerThreeMachineDefaultable[MachineInfo] = PerThreeMachineDefaultable()
 
         # Similar to coredata.compilers, but lower level in that there is no
         # meta data, only names/paths.
@@ -605,7 +631,7 @@ class Environment:
         #
         # Note that order matters because of 'buildtype', if it is after
         # 'optimization' and 'debug' keys, it override them.
-        self.options: T.MutableMapping[OptionKey, str] = collections.OrderedDict()
+        self.options: T.MutableMapping[OptionKey, T.Union[str, T.List[str]]] = collections.OrderedDict()
 
         ## Read in native file(s) to override build machine configuration
 
@@ -614,7 +640,7 @@ class Environment:
             binaries.build = BinaryTable(config.get('binaries', {}))
             properties.build = Properties(config.get('properties', {}))
             cmakevars.build = CMakeVariables(config.get('cmake', {}))
-            self.load_machine_file_options(config, properties.build, MachineChoice.BUILD)
+            self._load_machine_file_options(config, properties.build, MachineChoice.BUILD)
 
         ## Read in cross file(s) to override host machine configuration
 
@@ -632,7 +658,7 @@ class Environment:
             for key, value in list(self.options.items()):
                 if self.coredata.is_per_machine_option(key):
                     self.options[key.as_build()] = value
-            self.load_machine_file_options(config, properties.host, MachineChoice.HOST)
+            self._load_machine_file_options(config, properties.host, MachineChoice.HOST)
         else:
             # IF we aren't cross compiling, but we hav ea native file, the
             # native file is for the host. This is due to an mismatch between
@@ -652,7 +678,9 @@ class Environment:
         self.options.update(options.cmd_line_options)
 
         # Take default value from env if not set in cross/native files or command line.
-        self.set_default_options_from_env()
+        self._set_default_options_from_env()
+        self._set_default_binaries_from_env()
+        self._set_default_properties_from_env()
 
         # Warn if the user is using two different ways of setting build-type
         # options that override each other
@@ -718,7 +746,7 @@ class Environment:
         self.default_pkgconfig = ['pkg-config']
         self.wrap_resolver = None
 
-    def load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None:
+    def _load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None:
         """Read the contents of a Machine file and put it in the options store."""
         paths = config.get('paths')
         if paths:
@@ -749,23 +777,90 @@ class Environment:
                     key = OptionKey.from_string(k).evolve(subproject=subproject)
                     self.options[key] = v
 
-    def set_default_options_from_env(self) -> None:
-        for for_machine in MachineChoice:
-            for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path')]:
-                p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar)
-                if p_env_pair is not None:
-                    _, p_env = p_env_pair
-
-                    # PKG_CONFIG_PATH may contain duplicates, which must be
-                    # removed, else a duplicates-in-array-option warning arises.
+    def _set_default_options_from_env(self) -> None:
+        opts: T.List[T.Tuple[str, str]] = (
+            [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] +
+            [
+                ('PKG_CONFIG_PATH', 'pkg_config_path'),
+                ('CMAKE_PREFIX_PATH', 'cmake_prefix_path'),
+                ('LDFLAGS', 'ldflags'),
+                ('CPPFLAGS', 'cppflags'),
+            ]
+        )
+
+        for (evar, keyname), for_machine in itertools.product(opts, MachineChoice):
+            p_env = _get_env_var(for_machine, self.is_cross_build(), evar)
+            if p_env is not None:
+                # these may contain duplicates, which must be removed, else
+                # a duplicates-in-array-option warning arises.
+                if keyname == 'cmake_prefix_path':
+                    if self.machines[for_machine].is_windows():
+                        # Cannot split on ':' on Windows because its in the drive letter
+                        _p_env = p_env.split(os.pathsep)
+                    else:
+                        # https://github.com/mesonbuild/meson/issues/7294
+                        _p_env = re.split(r':|;', p_env)
+                    p_list = list(mesonlib.OrderedSet(_p_env))
+                elif keyname == 'pkg_config_path':
                     p_list = list(mesonlib.OrderedSet(p_env.split(':')))
-                    # Take env vars only on first invocation, if the env changes when
-                    # reconfiguring it gets ignored.
-                    # FIXME: We should remember if we took the value from env to warn
-                    # if it changes on future invocations.
-                    if self.first_invocation:
-                        key = OptionKey(keyname, machine=for_machine)
-                        self.options.setdefault(key, p_list)
+                else:
+                    p_list = split_args(p_env)
+                p_list = [e for e in p_list if e]  # filter out any empty eelemnts
+
+                # Take env vars only on first invocation, if the env changes when
+                # reconfiguring it gets ignored.
+                # FIXME: We should remember if we took the value from env to warn
+                # if it changes on future invocations.
+                if self.first_invocation:
+                    if keyname == 'ldflags':
+                        key = OptionKey('link_args', machine=for_machine, lang='c')  # needs a language to initialize properly
+                        for lang in compilers.compilers.LANGUAGES_USING_LDFLAGS:
+                            key = key.evolve(lang=lang)
+                            v = mesonlib.listify(self.options.get(key, []))
+                            self.options.setdefault(key, v + p_list)
+                    elif keyname == 'cppflags':
+                        key = OptionKey('args', machine=for_machine, lang='c')
+                        for lang in compilers.compilers.LANGUAGES_USING_CPPFLAGS:
+                            key = key.evolve(lang=lang)
+                            v = mesonlib.listify(self.options.get(key, []))
+                            self.options.setdefault(key, v + p_list)
+                    else:
+                        key = OptionKey.from_string(keyname).evolve(machine=for_machine)
+                        v = mesonlib.listify(self.options.get(key, []))
+                        self.options.setdefault(key, v + p_list)
+
+    def _set_default_binaries_from_env(self) -> None:
+        """Set default binaries from the environment.
+
+        For example, pkg-config can be set via PKG_CONFIG, or in the machine
+        file. We want to set the default to the env variable.
+        """
+        opts = itertools.chain(envconfig.DEPRECATED_ENV_PROG_MAP.items(),
+                               envconfig.ENV_VAR_PROG_MAP.items())
+
+        for (name, evar), for_machine in itertools.product(opts, MachineChoice):
+            p_env = _get_env_var(for_machine, self.is_cross_build(), evar)
+            if p_env is not None:
+                self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env))
+
+    def _set_default_properties_from_env(self) -> None:
+        """Properties which can alkso be set from the environment."""
+        # name, evar, split
+        opts: T.List[T.Tuple[str, T.List[str], bool]] = [
+            ('boost_includedir', ['BOOST_INCLUDEDIR'], False),
+            ('boost_librarydir', ['BOOST_LIBRARYDIR'], False),
+            ('boost_root', ['BOOST_ROOT', 'BOOSTROOT'], True),
+        ]
+
+        for (name, evars, split), for_machine in itertools.product(opts, MachineChoice):
+            for evar in evars:
+                p_env = _get_env_var(for_machine, self.is_cross_build(), evar)
+                if p_env is not None:
+                    if split:
+                        self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep))
+                    else:
+                        self.properties[for_machine].properties.setdefault(name, p_env)
+                    break
 
     def create_new_coredata(self, options: 'argparse.Namespace') -> None:
         # WARNING: Don't use any values from coredata in __init__. It gets
@@ -816,11 +911,8 @@ class Environment:
     def is_library(self, fname):
         return is_library(fname)
 
-    def lookup_binary_entry(self, for_machine: MachineChoice, name: str):
-        return self.binaries[for_machine].lookup_entry(
-            for_machine,
-            self.is_cross_build(),
-            name)
+    def lookup_binary_entry(self, for_machine: MachineChoice, name: str) -> T.List[str]:
+        return self.binaries[for_machine].lookup_entry(name)
 
     @staticmethod
     def get_gnu_compiler_defines(compiler):

mesonbuild/linkers.py

@@ -18,12 +18,11 @@ import typing as T
 
 from . import mesonlib
 from .arglist import CompilerArgs
-from .envconfig import get_env_var
 
 if T.TYPE_CHECKING:
     from .coredata import KeyedOptionDictType
-    from .envconfig import MachineChoice
     from .environment import Environment
+    from .mesonlib import MachineChoice
 
 
 class StaticLinker:
@@ -301,21 +300,7 @@ def evaluate_rpath(p: str, build_dir: str, from_dir: str) -> str:
     else:
         return os.path.relpath(os.path.join(build_dir, p), os.path.join(build_dir, from_dir))
 
-
-class LinkerEnvVarsMixin(metaclass=abc.ABCMeta):
-
-    """Mixin reading LDFLAGS from the environment."""
-
-    @staticmethod
-    def get_args_from_envvars(for_machine: mesonlib.MachineChoice,
-                              is_cross: bool) -> T.List[str]:
-        raw_value = get_env_var(for_machine, is_cross, 'LDFLAGS')
-        if raw_value is not None:
-            return mesonlib.split_args(raw_value)
-        else:
-            return []
-
-class DynamicLinker(LinkerEnvVarsMixin, metaclass=abc.ABCMeta):
+class DynamicLinker(metaclass=abc.ABCMeta):
 
     """Base class for dynamic linkers."""
 

mesonbuild/modules/unstable_external_project.py

@@ -23,7 +23,7 @@ from ..mesonlib import (MesonException, Popen_safe, MachineChoice,
 from ..interpreterbase import InterpreterObject, InterpreterException, FeatureNew
 from ..interpreterbase import stringArgs, permittedKwargs
 from ..interpreter import Interpreter, DependencyHolder, InstallDir
-from ..compilers.compilers import cflags_mapping, cexe_mapping
+from ..compilers.compilers import CFLAGS_MAPPING, CEXE_MAPPING
 from ..dependencies.base import InternalDependency, PkgConfigDependency
 from ..environment import Environment
 from ..mesonlib import OptionKey
@@ -110,11 +110,11 @@ class ExternalProject(InterpreterObject):
         link_args = []
         self.run_env = os.environ.copy()
         for lang, compiler in self.env.coredata.compilers[MachineChoice.HOST].items():
-            if any(lang not in i for i in (cexe_mapping, cflags_mapping)):
+            if any(lang not in i for i in (CEXE_MAPPING, CFLAGS_MAPPING)):
                 continue
             cargs = self.env.coredata.get_external_args(MachineChoice.HOST, lang)
-            self.run_env[cexe_mapping[lang]] = self._quote_and_join(compiler.get_exelist())
-            self.run_env[cflags_mapping[lang]] = self._quote_and_join(cargs)
+            self.run_env[CEXE_MAPPING[lang]] = self._quote_and_join(compiler.get_exelist())
+            self.run_env[CFLAGS_MAPPING[lang]] = self._quote_and_join(cargs)
             if not link_exelist:
                 link_exelist = compiler.get_linker_exelist()
                 link_args = self.env.coredata.get_external_link_args(MachineChoice.HOST, lang)

run_unittests.py

@@ -2606,8 +2606,10 @@ class AllPlatformTests(BasePlatformTests):
                 if is_osx():
                     self.assertIsInstance(cc.linker, mesonbuild.linkers.AppleDynamicLinker)
                 elif is_windows():
-                    # This is clang, not clang-cl
-                    self.assertIsInstance(cc.linker, mesonbuild.linkers.MSVCDynamicLinker)
+                    # This is clang, not clang-cl. This can be either an
+                    # ld-like linker of link.exe-like linker (usually the
+                    # former for msys2, the latter otherwise)
+                    self.assertIsInstance(cc.linker, (mesonbuild.linkers.MSVCDynamicLinker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin))
                 else:
                     self.assertIsInstance(cc.linker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin)
             if isinstance(cc, intel):
@@ -2635,17 +2637,16 @@ class AllPlatformTests(BasePlatformTests):
             # something like `ccache gcc -pipe` or `distcc ccache gcc` works.
             wrapper = os.path.join(testdir, 'compiler wrapper.py')
             wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG']
-            wrappercc_s = ''
-            for w in wrappercc:
-                wrappercc_s += quote_arg(w) + ' '
-            os.environ[evar] = wrappercc_s
-            wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST)
+            os.environ[evar] = ' '.join(quote_arg(w) for w in wrappercc)
+
             # Check static linker too
             wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args()
-            wrapperlinker_s = ''
-            for w in wrapperlinker:
-                wrapperlinker_s += quote_arg(w) + ' '
-            os.environ['AR'] = wrapperlinker_s
+            os.environ['AR'] = ' '.join(quote_arg(w) for w in wrapperlinker)
+
+            # Need a new env to re-run environment loading
+            env = get_fake_env(testdir, self.builddir, self.prefix)
+
+            wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST)
             wlinker = env.detect_static_linker(wcc)
             # Pop it so we don't use it for the next detection
             evalue = os.environ.pop('AR')
@@ -5375,7 +5376,7 @@ class FailureTests(BasePlatformTests):
     def test_boost_BOOST_ROOT_dependency(self):
         # Test BOOST_ROOT; can be run even if Boost is found or not
         self.assertMesonRaises("dependency('boost')",
-                               "(BOOST_ROOT.*absolute|{})".format(self.dnf),
+                               "(boost_root.*absolute|{})".format(self.dnf),
                                override_envvars = {'BOOST_ROOT': 'relative/path'})
 
     def test_dependency_invalid_method(self):
@@ -5687,12 +5688,12 @@ class WindowsTests(BasePlatformTests):
     def _check_ld(self, name: str, lang: str, expected: str) -> None:
         if not shutil.which(name):
             raise unittest.SkipTest('Could not find {}.'.format(name))
-        envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]]
+        envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]]
 
         # Also test a deprecated variable if there is one.
-        if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP:
+        if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP:
             envvars.append(
-                mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]])
+                mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld'])
 
         for envvar in envvars:
             with mock.patch.dict(os.environ, {envvar: name}):
@@ -7291,12 +7292,12 @@ class LinuxlikeTests(BasePlatformTests):
             raise unittest.SkipTest('Solaris currently cannot override the linker.')
         if not shutil.which(check):
             raise unittest.SkipTest('Could not find {}.'.format(check))
-        envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]]
+        envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]]
 
         # Also test a deprecated variable if there is one.
-        if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP:
+        if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP:
             envvars.append(
-                mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]])
+                mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld'])
 
         for envvar in envvars:
             with mock.patch.dict(os.environ, {envvar: name}):
@@ -8272,7 +8273,7 @@ class NativeFileTests(BasePlatformTests):
             return 'gfortran', 'gcc'
         self.helper_for_compiler('fortran', cb)
 
-    def _single_implementation_compiler(self, lang, binary, version_str, version):
+    def _single_implementation_compiler(self, lang: str, binary: str, version_str: str, version: str) -> None:
         """Helper for languages with a single (supported) implementation.
 
         Builds a wrapper around the compiler to override the version.
@@ -8281,7 +8282,7 @@ class NativeFileTests(BasePlatformTests):
         env = get_fake_env()
         getter = getattr(env, 'detect_{}_compiler'.format(lang))
         getter = functools.partial(getter, MachineChoice.HOST)
-        env.binaries.host.binaries[lang] = wrapper
+        env.binaries.host.binaries[lang] = [wrapper]
         compiler = getter()
         self.assertEqual(compiler.version, version)
 
@@ -8308,7 +8309,7 @@ class NativeFileTests(BasePlatformTests):
             'swiftc', version='Swift 1.2345', outfile='stderr',
             extra_args={'Xlinker': 'macosx_version. PROJECT:ld - 1.2.3'})
         env = get_fake_env()
-        env.binaries.host.binaries['swift'] = wrapper
+        env.binaries.host.binaries['swift'] = [wrapper]
         compiler = env.detect_swift_compiler(MachineChoice.HOST)
         self.assertEqual(compiler.version, '1.2345')
 
@@ -9345,7 +9346,7 @@ def unset_envs():
     # For unit tests we must fully control all command lines
     # so that there are no unexpected changes coming from the
     # environment, for example when doing a package build.
-    varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.cflags_mapping.values())
+    varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.CFLAGS_MAPPING.values())
     for v in varnames:
         if v in os.environ:
             del os.environ[v]