From 9f343212e071cfee99e0be76de0f61bec9a6d5f2 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Mon, 11 Jan 2021 11:09:16 -0800 Subject: [PATCH 01/14] unittests: Clang on windows can use either ld or link.exe it generally uses the ld style linkers with msys2 and link.exe style linkers otherwise, but anything's possible. --- run_unittests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run_unittests.py b/run_unittests.py index 27c067711..5118b0096 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2606,8 +2606,10 @@ class AllPlatformTests(BasePlatformTests): if is_osx(): self.assertIsInstance(cc.linker, mesonbuild.linkers.AppleDynamicLinker) elif is_windows(): - # This is clang, not clang-cl - self.assertIsInstance(cc.linker, mesonbuild.linkers.MSVCDynamicLinker) + # This is clang, not clang-cl. This can be either an + # ld-like linker of link.exe-like linker (usually the + # former for msys2, the latter otherwise) + self.assertIsInstance(cc.linker, (mesonbuild.linkers.MSVCDynamicLinker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin)) else: self.assertIsInstance(cc.linker, mesonbuild.linkers.GnuLikeDynamicLinkerMixin) if isinstance(cc, intel): From a2055ad18bba5704c9cbbe211ed4e46549be05f8 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 14:09:56 -0800 Subject: [PATCH 02/14] import MachineChoice from mesonlib there are a couple of places importing it from envconfig, which is not correct. It's defined in mesonlib, and then imported into envconfig. --- mesonbuild/dependencies/hdf5.py | 2 +- mesonbuild/linkers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/dependencies/hdf5.py b/mesonbuild/dependencies/hdf5.py index 21f4e71f5..ad28975a6 100644 --- a/mesonbuild/dependencies/hdf5.py +++ b/mesonbuild/dependencies/hdf5.py @@ -30,8 +30,8 @@ import typing as T if T.TYPE_CHECKING: from .base import Dependency - from ..envconfig import MachineChoice from ..environment import Environment + from ..mesonlib import MachineChoice class HDF5PkgConfigDependency(PkgConfigDependency): diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 141c8fdaa..8dae94ce6 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -22,8 +22,8 @@ from .envconfig import get_env_var if T.TYPE_CHECKING: from .coredata import KeyedOptionDictType - from .envconfig import MachineChoice from .environment import Environment + from .mesonlib import MachineChoice class StaticLinker: From 284b89c3217688ac09b79463a20e4099d7c158b9 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 14:10:59 -0800 Subject: [PATCH 03/14] dependencies/mpi: Add a type annotation Some change in this series causes mypy to notice that this isn't annotated, and it makes a wrong assumption. --- mesonbuild/dependencies/mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/dependencies/mpi.py b/mesonbuild/dependencies/mpi.py index 2e1e76446..d4b324cca 100644 --- a/mesonbuild/dependencies/mpi.py +++ b/mesonbuild/dependencies/mpi.py @@ -106,7 +106,7 @@ class _MPIConfigToolDependency(ConfigToolDependency): Drop -O2 and everything that is not needed. """ result = [] - multi_args = ('-I', ) + multi_args: T.Tuple[str, ...] = ('-I', ) if self.language == 'fortran': fc = self.env.coredata.compilers[self.for_machine]['fortran'] multi_args += fc.get_module_incdir_args() From 3949c2a0e0a5fbd9ede8e7a4607220e58934637e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:26:03 -0800 Subject: [PATCH 04/14] move CMAKE_PREFIX_PATH env var handling to environment This causes the variable to be read up front and stored, rather than be re-read on each invocation of meson. This does have two slight behavioral changes. First is the obvious one that changing the variable between `meson --reconfigure` invocations has no effect. This is the way PKG_CONFIG_PATH already works. The second change is that CMAKE_PREFIX_PATH the env var is no longer appended to the values set in the machine file or on the command line, and is instead replaced by them. CMAKE_PREFIX_PATH is the only env var in meson that works this way, every other one is replaced not appended, so while this is a behavioral change, I also think its a bug fix. --- mesonbuild/cmake/executor.py | 18 ------------------ mesonbuild/environment.py | 26 ++++++++++++++++++++------ 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/mesonbuild/cmake/executor.py b/mesonbuild/cmake/executor.py index 674b854c8..e4b85de81 100644 --- a/mesonbuild/cmake/executor.py +++ b/mesonbuild/cmake/executor.py @@ -24,7 +24,6 @@ import os from .. import mlog from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey -from ..envconfig import get_env_var if T.TYPE_CHECKING: from ..environment import Environment @@ -63,23 +62,6 @@ class CMakeExecutor: return self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value - env_pref_path_raw = get_env_var( - self.for_machine, - self.environment.is_cross_build(), - 'CMAKE_PREFIX_PATH') - if env_pref_path_raw is not None: - env_pref_path = [] # type: T.List[str] - if is_windows(): - # Cannot split on ':' on Windows because its in the drive letter - env_pref_path = env_pref_path_raw.split(os.pathsep) - else: - # https://github.com/mesonbuild/meson/issues/7294 - env_pref_path = re.split(r':|;', env_pref_path_raw) - env_pref_path = [x for x in env_pref_path if x] # Filter out empty strings - if not self.prefix_paths: - self.prefix_paths = [] - self.prefix_paths += env_pref_path - if self.prefix_paths: self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))] diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 59675ff9a..7555be53b 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -1,4 +1,4 @@ -# Copyright 2012-2016 The Meson development team +# Copyright 2012-2020 The Meson development team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -579,7 +579,7 @@ class Environment: # Stores machine infos, the only *three* machine one because we have a # target machine info on for the user (Meson never cares about the # target machine.) - machines = PerThreeMachineDefaultable() # type: PerMachineDefaultable[MachineInfo] + machines: PerThreeMachineDefaultable[MachineInfo] = PerThreeMachineDefaultable() # Similar to coredata.compilers, but lower level in that there is no # meta data, only names/paths. @@ -751,14 +751,28 @@ class Environment: def set_default_options_from_env(self) -> None: for for_machine in MachineChoice: - for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path')]: + for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path'), + ('CMAKE_PREFIX_PATH', 'cmake_prefix_path')]: p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) if p_env_pair is not None: _, p_env = p_env_pair - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. - p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + # these may contain duplicates, which must be removed, else + # a duplicates-in-array-option warning arises. + if keyname == 'cmake_prefix_path': + if self.machines[for_machine].is_windows(): + # Cannot split on ':' on Windows because its in the drive letter + _p_env = p_env.split(os.pathsep) + else: + # https://github.com/mesonbuild/meson/issues/7294 + _p_env = re.split(r':|;', p_env) + p_list = list(mesonlib.OrderedSet(_p_env)) + elif keyname == 'pkg_config_path': + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + else: + raise RuntimeError('Should be unreachable') + p_list = [e for e in p_list if e] # filter out any empty eelemnts + # Take env vars only on first invocation, if the env changes when # reconfiguring it gets ignored. # FIXME: We should remember if we took the value from env to warn From 7248a64750fc0748f7d86b65e03172f00b515690 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Thu, 10 Dec 2020 18:01:02 -0800 Subject: [PATCH 05/14] dependencies: Don't read PKG_CONFIG_PATH from the env again We already read this in, don't read it again. Just rely on the value we have stored. --- mesonbuild/dependencies/base.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index 9ddf6dba7..9330e46b3 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -32,7 +32,6 @@ from pathlib import Path, PurePath from .. import mlog from .. import mesonlib from ..compilers import clib_langs -from ..envconfig import get_env_var from ..environment import Environment, MachineInfo from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException, CMakeToolchain, CMakeExecScope, check_cmake_args from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine @@ -317,7 +316,7 @@ class HasNativeKwarg: return MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST class ExternalDependency(Dependency, HasNativeKwarg): - def __init__(self, type_name, environment, kwargs, language: T.Optional[str] = None): + def __init__(self, type_name, environment: Environment, kwargs, language: T.Optional[str] = None): Dependency.__init__(self, type_name, kwargs) self.env = environment self.name = type_name # default @@ -784,14 +783,7 @@ class PkgConfigDependency(ExternalDependency): # # Only prefix_libpaths are reordered here because there should not be # too many system_libpaths to cause library version issues. - pkg_config_path = get_env_var( - self.for_machine, - self.env.is_cross_build(), - 'PKG_CONFIG_PATH') - if pkg_config_path: - pkg_config_path = pkg_config_path.split(os.pathsep) - else: - pkg_config_path = [] + pkg_config_path: T.List[str] = self.env.coredata.options[OptionKey('pkg_config_path', machine=self.for_machine)].value pkg_config_path = self._convert_mingw_paths(pkg_config_path) prefix_libpaths = sort_libpaths(prefix_libpaths, pkg_config_path) system_libpaths = OrderedSet() From 4580433b8255b9f69dfbd39a28c21bc1ec785bcf Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:33:11 -0800 Subject: [PATCH 06/14] rename cflags_mapping to CFLAGS_MAPPING This is PEP8 convention for a const variable. Also, make the type Mapping, which doesn't have mutation methods. This means mypy will warn us if someone tries to change this. --- mesonbuild/compilers/compilers.py | 32 +++++++++++-------- .../modules/unstable_external_project.py | 8 ++--- run_unittests.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 234ce06e9..41e1036b2 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -91,18 +91,22 @@ languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$') # Environment variables that each lang uses. -cflags_mapping = {'c': 'CFLAGS', - 'cpp': 'CXXFLAGS', - 'cuda': 'CUFLAGS', - 'objc': 'OBJCFLAGS', - 'objcpp': 'OBJCXXFLAGS', - 'fortran': 'FFLAGS', - 'd': 'DFLAGS', - 'vala': 'VALAFLAGS', - 'rust': 'RUSTFLAGS'} # type: T.Dict[str, str] - -cexe_mapping = {'c': 'CC', - 'cpp': 'CXX'} +CFLAGS_MAPPING: T.Mapping[str, str] = { + 'c': 'CFLAGS', + 'cpp': 'CXXFLAGS', + 'cuda': 'CUFLAGS', + 'objc': 'OBJCFLAGS', + 'objcpp': 'OBJCXXFLAGS', + 'fortran': 'FFLAGS', + 'd': 'DFLAGS', + 'vala': 'VALAFLAGS', + 'rust': 'RUSTFLAGS', +} + +CEXE_MAPPING: T.Mapping = { + 'c': 'CC', + 'cpp': 'CXX', +} # All these are only for C-linkable languages; see `clink_langs` above. @@ -1207,13 +1211,13 @@ def get_args_from_envvars(lang: str, Returns a tuple of (compile_flags, link_flags) for the specified language from the inherited environment """ - if lang not in cflags_mapping: + if lang not in CFLAGS_MAPPING: return [], [] compile_flags = [] # type: T.List[str] link_flags = [] # type: T.List[str] - env_compile_flags = get_env_var(for_machine, is_cross, cflags_mapping[lang]) + env_compile_flags = get_env_var(for_machine, is_cross, CFLAGS_MAPPING[lang]) if env_compile_flags is not None: compile_flags += split_args(env_compile_flags) diff --git a/mesonbuild/modules/unstable_external_project.py b/mesonbuild/modules/unstable_external_project.py index 7249078ac..809b590bd 100644 --- a/mesonbuild/modules/unstable_external_project.py +++ b/mesonbuild/modules/unstable_external_project.py @@ -23,7 +23,7 @@ from ..mesonlib import (MesonException, Popen_safe, MachineChoice, from ..interpreterbase import InterpreterObject, InterpreterException, FeatureNew from ..interpreterbase import stringArgs, permittedKwargs from ..interpreter import Interpreter, DependencyHolder, InstallDir -from ..compilers.compilers import cflags_mapping, cexe_mapping +from ..compilers.compilers import CFLAGS_MAPPING, CEXE_MAPPING from ..dependencies.base import InternalDependency, PkgConfigDependency from ..environment import Environment from ..mesonlib import OptionKey @@ -110,11 +110,11 @@ class ExternalProject(InterpreterObject): link_args = [] self.run_env = os.environ.copy() for lang, compiler in self.env.coredata.compilers[MachineChoice.HOST].items(): - if any(lang not in i for i in (cexe_mapping, cflags_mapping)): + if any(lang not in i for i in (CEXE_MAPPING, CFLAGS_MAPPING)): continue cargs = self.env.coredata.get_external_args(MachineChoice.HOST, lang) - self.run_env[cexe_mapping[lang]] = self._quote_and_join(compiler.get_exelist()) - self.run_env[cflags_mapping[lang]] = self._quote_and_join(cargs) + self.run_env[CEXE_MAPPING[lang]] = self._quote_and_join(compiler.get_exelist()) + self.run_env[CFLAGS_MAPPING[lang]] = self._quote_and_join(cargs) if not link_exelist: link_exelist = compiler.get_linker_exelist() link_args = self.env.coredata.get_external_link_args(MachineChoice.HOST, lang) diff --git a/run_unittests.py b/run_unittests.py index 5118b0096..2f102b471 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -9347,7 +9347,7 @@ def unset_envs(): # For unit tests we must fully control all command lines # so that there are no unexpected changes coming from the # environment, for example when doing a package build. - varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.cflags_mapping.values()) + varnames = ['CPPFLAGS', 'LDFLAGS'] + list(mesonbuild.compilers.compilers.CFLAGS_MAPPING.values()) for v in varnames: if v in os.environ: del os.environ[v] From f202da0689795ba4330581e69a879f089e169f6c Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 10:56:44 -0800 Subject: [PATCH 07/14] use PEP8 style naming for LANGUAGES_USING_* as well --- mesonbuild/backend/backends.py | 4 ++-- mesonbuild/compilers/__init__.py | 2 +- mesonbuild/compilers/compilers.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mesonbuild/backend/backends.py b/mesonbuild/backend/backends.py index 9bb870ca7..6dad1894d 100644 --- a/mesonbuild/backend/backends.py +++ b/mesonbuild/backend/backends.py @@ -29,7 +29,7 @@ from .. import build from .. import dependencies from .. import mesonlib from .. import mlog -from ..compilers import languages_using_ldflags +from ..compilers import LANGUAGES_USING_LDFLAGS from ..mesonlib import ( File, MachineChoice, MesonException, OptionType, OrderedSet, OptionOverrideProxy, classify_unity_sources, unholder, OptionKey @@ -480,7 +480,7 @@ class Backend: def get_external_rpath_dirs(self, target): dirs = set() args = [] - for lang in languages_using_ldflags: + for lang in LANGUAGES_USING_LDFLAGS: try: args.extend(self.environment.coredata.get_external_link_args(target.for_machine, lang)) except Exception: diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py index ec5d30f00..bda608642 100644 --- a/mesonbuild/compilers/__init__.py +++ b/mesonbuild/compilers/__init__.py @@ -127,7 +127,7 @@ from .compilers import ( is_library, is_known_suffix, lang_suffixes, - languages_using_ldflags, + LANGUAGES_USING_LDFLAGS, sort_clink, ) from .c import ( diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 41e1036b2..23a3bc5cc 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -85,9 +85,9 @@ clink_suffixes += ('h', 'll', 's') all_suffixes = set(itertools.chain(*lang_suffixes.values(), clink_suffixes)) # type: T.Set[str] # Languages that should use LDFLAGS arguments when linking. -languages_using_ldflags = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] +LANGUAGES_USING_LDFLAGS = {'objcpp', 'cpp', 'objc', 'c', 'fortran', 'd', 'cuda'} # type: T.Set[str] # Languages that should use CPPFLAGS arguments when linking. -languages_using_cppflags = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] +LANGUAGES_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcpp'} # type: T.Set[str] soregex = re.compile(r'.*\.so(\.[0-9]+)?(\.[0-9]+)?(\.[0-9]+)?$') # Environment variables that each lang uses. @@ -1222,7 +1222,7 @@ def get_args_from_envvars(lang: str, compile_flags += split_args(env_compile_flags) # Link flags (same for all languages) - if lang in languages_using_ldflags: + if lang in LANGUAGES_USING_LDFLAGS: link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross) if use_linker_args: # When the compiler is used as a wrapper around the linker (such as @@ -1232,7 +1232,7 @@ def get_args_from_envvars(lang: str, link_flags = compile_flags + link_flags # Pre-processor flags for certain languages - if lang in languages_using_cppflags: + if lang in LANGUAGES_USING_CPPFLAGS: env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS') if env_preproc_flags is not None: compile_flags += split_args(env_preproc_flags) From 4b0b44aface1cd9ed073733dd80b040b5aaf4c99 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Tue, 8 Dec 2020 11:08:37 -0800 Subject: [PATCH 08/14] move handling of CFLAGS and friends to environment This has a bunch of nice features. It obviously centralizes everything, which is nice. It also means that env is only re-read at `meson --wipe`, not `meson --reconfigure`. And it's going to allow more cleanups. --- mesonbuild/compilers/c.py | 3 +- mesonbuild/compilers/compilers.py | 75 ++++++---------------------- mesonbuild/compilers/cpp.py | 3 +- mesonbuild/coredata.py | 2 +- mesonbuild/environment.py | 83 ++++++++++++++++++++----------- mesonbuild/linkers.py | 17 +------ 6 files changed, 72 insertions(+), 111 deletions(-) diff --git a/mesonbuild/compilers/c.py b/mesonbuild/compilers/c.py index 311e65a41..4a55b462d 100644 --- a/mesonbuild/compilers/c.py +++ b/mesonbuild/compilers/c.py @@ -17,7 +17,6 @@ import typing as T from .. import coredata from ..mesonlib import MachineChoice, MesonException, mlog, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .c_function_attributes import C_FUNC_ATTRIBUTES from .mixins.clike import CLikeCompiler from .mixins.ccrx import CcrxCompiler @@ -195,7 +194,7 @@ class AppleClangCCompiler(ClangCCompiler): _C2X_VERSION = '>=11.0.0' -class EmscriptenCCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCCompiler): +class EmscriptenCCompiler(EmscriptenMixin, ClangCCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py index 23a3bc5cc..cf9f35baf 100644 --- a/mesonbuild/compilers/compilers.py +++ b/mesonbuild/compilers/compilers.py @@ -22,13 +22,9 @@ from functools import lru_cache from .. import coredata from .. import mlog from .. import mesonlib -from ..linkers import LinkerEnvVarsMixin from ..mesonlib import ( EnvironmentException, MachineChoice, MesonException, - Popen_safe, split_args, LibType, TemporaryDirectoryWinProof, OptionKey, -) -from ..envconfig import ( - get_env_var + Popen_safe, LibType, TemporaryDirectoryWinProof, OptionKey, ) from ..arglist import CompilerArgs @@ -105,7 +101,7 @@ CFLAGS_MAPPING: T.Mapping[str, str] = { CEXE_MAPPING: T.Mapping = { 'c': 'CC', - 'cpp': 'CXX', + 'cpp': 'CXX', } # All these are only for C-linkable languages; see `clink_langs` above. @@ -591,11 +587,6 @@ class Compiler(metaclass=abc.ABCMeta): """ return [] - def get_linker_args_from_envvars(self, - for_machine: MachineChoice, - is_cross: bool) -> T.List[str]: - return self.linker.get_args_from_envvars(for_machine, is_cross) - def get_options(self) -> 'KeyedOptionDictType': return {} @@ -1203,68 +1194,32 @@ class Compiler(metaclass=abc.ABCMeta): def get_prelink_args(self, prelink_name: str, obj_list: T.List[str]) -> T.List[str]: raise EnvironmentException('{} does not know how to do prelinking.'.format(self.id)) -def get_args_from_envvars(lang: str, - for_machine: MachineChoice, - is_cross: bool, - use_linker_args: bool) -> T.Tuple[T.List[str], T.List[str]]: - """ - Returns a tuple of (compile_flags, link_flags) for the specified language - from the inherited environment - """ - if lang not in CFLAGS_MAPPING: - return [], [] - - compile_flags = [] # type: T.List[str] - link_flags = [] # type: T.List[str] - - env_compile_flags = get_env_var(for_machine, is_cross, CFLAGS_MAPPING[lang]) - if env_compile_flags is not None: - compile_flags += split_args(env_compile_flags) - - # Link flags (same for all languages) - if lang in LANGUAGES_USING_LDFLAGS: - link_flags += LinkerEnvVarsMixin.get_args_from_envvars(for_machine, is_cross) - if use_linker_args: - # When the compiler is used as a wrapper around the linker (such as - # with GCC and Clang), the compile flags can be needed while linking - # too. This is also what Autotools does. However, we don't want to do - # this when the linker is stand-alone such as with MSVC C/C++, etc. - link_flags = compile_flags + link_flags - - # Pre-processor flags for certain languages - if lang in LANGUAGES_USING_CPPFLAGS: - env_preproc_flags = get_env_var(for_machine, is_cross, 'CPPFLAGS') - if env_preproc_flags is not None: - compile_flags += split_args(env_preproc_flags) - - return compile_flags, link_flags - def get_global_options(lang: str, comp: T.Type[Compiler], for_machine: MachineChoice, - is_cross: bool) -> 'KeyedOptionDictType': + env: 'Environment') -> 'KeyedOptionDictType': """Retreive options that apply to all compilers for a given language.""" description = 'Extra arguments passed to the {}'.format(lang) argkey = OptionKey('args', lang=lang, machine=for_machine) largkey = argkey.evolve('link_args') + + # We shouldn't need listify here, but until we have a separate + # linker-driver representation and can have that do the combine we have to + # do it htis way. + compile_args = mesonlib.listify(env.options.get(argkey, [])) + link_args = mesonlib.listify(env.options.get(largkey, [])) + + if comp.INVOKES_LINKER: + link_args = compile_args + link_args + opts: 'KeyedOptionDictType' = { argkey: coredata.UserArrayOption( description + ' compiler', - [], split_args=True, user_input=True, allow_dups=True), + compile_args, split_args=True, user_input=True, allow_dups=True), largkey: coredata.UserArrayOption( description + ' linker', - [], split_args=True, user_input=True, allow_dups=True), + link_args, split_args=True, user_input=True, allow_dups=True), } - # Get from env vars. - compile_args, link_args = get_args_from_envvars( - lang, - for_machine, - is_cross, - comp.INVOKES_LINKER) - - opts[argkey].set_value(compile_args) - opts[largkey].set_value(link_args) - return opts diff --git a/mesonbuild/compilers/cpp.py b/mesonbuild/compilers/cpp.py index 2e94e483e..ebe3a4f5d 100644 --- a/mesonbuild/compilers/cpp.py +++ b/mesonbuild/compilers/cpp.py @@ -21,7 +21,6 @@ from .. import coredata from .. import mlog from ..mesonlib import MesonException, MachineChoice, version_compare, OptionKey -from ..linkers import LinkerEnvVarsMixin from .compilers import ( gnu_winlibs, msvc_winlibs, @@ -256,7 +255,7 @@ class AppleClangCPPCompiler(ClangCPPCompiler): return ['-lc++'] -class EmscriptenCPPCompiler(EmscriptenMixin, LinkerEnvVarsMixin, ClangCPPCompiler): +class EmscriptenCPPCompiler(EmscriptenMixin, ClangCPPCompiler): def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoice, is_cross: bool, info: 'MachineInfo', exe_wrapper: T.Optional['ExternalProgram'] = None, linker: T.Optional['DynamicLinker'] = None, diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index cda0566ce..f2aba8022 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -759,7 +759,7 @@ class CoreData: for_machine: MachineChoice, env: 'Environment') -> None: """Add global language arguments that are needed before compiler/linker detection.""" from .compilers import compilers - options = compilers.get_global_options(lang, comp, for_machine, env.is_cross_build()) + options = compilers.get_global_options(lang, comp, for_machine, env) self.add_compiler_options(options, lang, for_machine, env) def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 7555be53b..12b9dbb49 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import os, platform, re, sys, shutil, subprocess import tempfile import shlex @@ -605,7 +606,7 @@ class Environment: # # Note that order matters because of 'buildtype', if it is after # 'optimization' and 'debug' keys, it override them. - self.options: T.MutableMapping[OptionKey, str] = collections.OrderedDict() + self.options: T.MutableMapping[OptionKey, T.Union[str, T.List[str]]] = collections.OrderedDict() ## Read in native file(s) to override build machine configuration @@ -750,36 +751,58 @@ class Environment: self.options[key] = v def set_default_options_from_env(self) -> None: - for for_machine in MachineChoice: - for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path'), - ('CMAKE_PREFIX_PATH', 'cmake_prefix_path')]: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair - - # these may contain duplicates, which must be removed, else - # a duplicates-in-array-option warning arises. - if keyname == 'cmake_prefix_path': - if self.machines[for_machine].is_windows(): - # Cannot split on ':' on Windows because its in the drive letter - _p_env = p_env.split(os.pathsep) - else: - # https://github.com/mesonbuild/meson/issues/7294 - _p_env = re.split(r':|;', p_env) - p_list = list(mesonlib.OrderedSet(_p_env)) - elif keyname == 'pkg_config_path': - p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + opts: T.List[T.Tuple[str, str]] = ( + [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] + + [ + ('PKG_CONFIG_PATH', 'pkg_config_path'), + ('CMAKE_PREFIX_PATH', 'cmake_prefix_path'), + ('LDFLAGS', 'ldflags'), + ('CPPFLAGS', 'cppflags'), + ] + ) + + for (evar, keyname), for_machine in itertools.product(opts, MachineChoice): + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + + # these may contain duplicates, which must be removed, else + # a duplicates-in-array-option warning arises. + if keyname == 'cmake_prefix_path': + if self.machines[for_machine].is_windows(): + # Cannot split on ':' on Windows because its in the drive letter + _p_env = p_env.split(os.pathsep) else: - raise RuntimeError('Should be unreachable') - p_list = [e for e in p_list if e] # filter out any empty eelemnts - - # Take env vars only on first invocation, if the env changes when - # reconfiguring it gets ignored. - # FIXME: We should remember if we took the value from env to warn - # if it changes on future invocations. - if self.first_invocation: - key = OptionKey(keyname, machine=for_machine) - self.options.setdefault(key, p_list) + # https://github.com/mesonbuild/meson/issues/7294 + _p_env = re.split(r':|;', p_env) + p_list = list(mesonlib.OrderedSet(_p_env)) + elif keyname == 'pkg_config_path': + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + else: + p_list = split_args(p_env) + p_list = [e for e in p_list if e] # filter out any empty eelemnts + + # Take env vars only on first invocation, if the env changes when + # reconfiguring it gets ignored. + # FIXME: We should remember if we took the value from env to warn + # if it changes on future invocations. + if self.first_invocation: + if keyname == 'ldflags': + key = OptionKey('link_args', machine=for_machine, lang='c') # needs a language to initialize properly + for lang in compilers.compilers.LANGUAGES_USING_LDFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + elif keyname == 'cppflags': + key = OptionKey('args', machine=for_machine, lang='c') + for lang in compilers.compilers.LANGUAGES_USING_CPPFLAGS: + key = key.evolve(lang=lang) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) + else: + key = OptionKey.from_string(keyname).evolve(machine=for_machine) + v = mesonlib.listify(self.options.get(key, [])) + self.options.setdefault(key, v + p_list) def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 8dae94ce6..86e6aaca5 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -18,7 +18,6 @@ import typing as T from . import mesonlib from .arglist import CompilerArgs -from .envconfig import get_env_var if T.TYPE_CHECKING: from .coredata import KeyedOptionDictType @@ -301,21 +300,7 @@ def evaluate_rpath(p: str, build_dir: str, from_dir: str) -> str: else: return os.path.relpath(os.path.join(build_dir, p), os.path.join(build_dir, from_dir)) - -class LinkerEnvVarsMixin(metaclass=abc.ABCMeta): - - """Mixin reading LDFLAGS from the environment.""" - - @staticmethod - def get_args_from_envvars(for_machine: mesonlib.MachineChoice, - is_cross: bool) -> T.List[str]: - raw_value = get_env_var(for_machine, is_cross, 'LDFLAGS') - if raw_value is not None: - return mesonlib.split_args(raw_value) - else: - return [] - -class DynamicLinker(LinkerEnvVarsMixin, metaclass=abc.ABCMeta): +class DynamicLinker(metaclass=abc.ABCMeta): """Base class for dynamic linkers.""" From 38c7a7590c25426dfd2d480d805d570d7e645096 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 11:29:43 -0800 Subject: [PATCH 09/14] pull env to program mappings out of BinaryType class These really aren't pivotal to that class, and they're used outside of it. In a follow up patch they're not going to be used inside it at all. --- mesonbuild/envconfig.py | 90 ++++++++++++++++++++--------------------- run_unittests.py | 8 ++-- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index db1041f81..642aab3c7 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -85,6 +85,49 @@ CPU_FAMILES_64_BIT = [ 'x86_64', ] +# Map from language identifiers to environment variables. +ENV_VAR_PROG_MAP: T.Mapping[str, str] = { + # Compilers + 'c': 'CC', + 'cpp': 'CXX', + 'cs': 'CSC', + 'd': 'DC', + 'fortran': 'FC', + 'objc': 'OBJC', + 'objcpp': 'OBJCXX', + 'rust': 'RUSTC', + 'vala': 'VALAC', + + # Linkers + 'c_ld': 'CC_LD', + 'cpp_ld': 'CXX_LD', + 'd_ld': 'DC_LD', + 'fortran_ld': 'FC_LD', + 'objc_ld': 'OBJC_LD', + 'objcpp_ld': 'OBJCXX_LD', + 'rust_ld': 'RUSTC_LD', + + # Binutils + 'strip': 'STRIP', + 'ar': 'AR', + 'windres': 'WINDRES', + + # Other tools + 'cmake': 'CMAKE', + 'qmake': 'QMAKE', + 'pkgconfig': 'PKG_CONFIG', + 'make': 'MAKE', +} + +# Deprecated environment variables mapped from the new variable to the old one +# Deprecated in 0.54.0 +DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = { + 'DC_LD': 'D_LD', + 'FC_LD': 'F_LD', + 'RUSTC_LD': 'RUST_LD', + 'OBJCXX_LD': 'OBJCPP_LD', +} + class CMakeSkipCompilerTest(Enum): ALWAYS = 'always' NEVER = 'never' @@ -363,49 +406,6 @@ class BinaryTable: 'Invalid type {!r} for binary {!r} in cross file' ''.format(command, name)) - # Map from language identifiers to environment variables. - evarMap = { - # Compilers - 'c': 'CC', - 'cpp': 'CXX', - 'cs': 'CSC', - 'd': 'DC', - 'fortran': 'FC', - 'objc': 'OBJC', - 'objcpp': 'OBJCXX', - 'rust': 'RUSTC', - 'vala': 'VALAC', - - # Linkers - 'c_ld': 'CC_LD', - 'cpp_ld': 'CXX_LD', - 'd_ld': 'DC_LD', - 'fortran_ld': 'FC_LD', - 'objc_ld': 'OBJC_LD', - 'objcpp_ld': 'OBJCXX_LD', - 'rust_ld': 'RUSTC_LD', - - # Binutils - 'strip': 'STRIP', - 'ar': 'AR', - 'windres': 'WINDRES', - - # Other tools - 'cmake': 'CMAKE', - 'qmake': 'QMAKE', - 'pkgconfig': 'PKG_CONFIG', - 'make': 'MAKE', - } # type: T.Dict[str, str] - - # Deprecated environment variables mapped from the new variable to the old one - # Deprecated in 0.54.0 - DEPRECATION_MAP = { - 'DC_LD': 'D_LD', - 'FC_LD': 'F_LD', - 'RUSTC_LD': 'RUST_LD', - 'OBJCXX_LD': 'OBJCPP_LD', - } # type: T.Dict[str, str] - @staticmethod def detect_ccache() -> T.List[str]: try: @@ -442,11 +442,11 @@ class BinaryTable: if raw_command is not None: command = mesonlib.stringlistify(raw_command) break # found - evar = self.evarMap.get(name) + evar = ENV_VAR_PROG_MAP.get(name) if evar is not None: raw_command = get_env_var(for_machine, is_cross, evar) if raw_command is None: - deprecated = self.DEPRECATION_MAP.get(evar) + deprecated = DEPRECATED_ENV_PROG_MAP.get(evar) if deprecated is not None: raw_command = get_env_var(for_machine, is_cross, deprecated) if raw_command is not None: diff --git a/run_unittests.py b/run_unittests.py index 2f102b471..cf78e3ef2 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5689,12 +5689,12 @@ class WindowsTests(BasePlatformTests): def _check_ld(self, name: str, lang: str, expected: str) -> None: if not shutil.which(name): raise unittest.SkipTest('Could not find {}.'.format(name)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if envvars[0] in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[envvars[0]]) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -7293,7 +7293,7 @@ class LinuxlikeTests(BasePlatformTests): raise unittest.SkipTest('Solaris currently cannot override the linker.') if not shutil.which(check): raise unittest.SkipTest('Could not find {}.'.format(check)) - envvars = [mesonbuild.envconfig.BinaryTable.evarMap['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.BinaryTable.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: From e7a5c75285ce63a7197cd82e893450eb9bb68b6c Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 12:30:19 -0800 Subject: [PATCH 10/14] Move BinaryTable environment lookups to Environment This means that all the env lookups are done once, at initial configure time. This has all of the expected advantages. --- mesonbuild/envconfig.py | 63 +++++++++++++-------------------------- mesonbuild/environment.py | 26 ++++++++++++---- run_unittests.py | 33 ++++++++++---------- 3 files changed, 57 insertions(+), 65 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 642aab3c7..69b667869 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -23,6 +23,9 @@ from pathlib import Path _T = T.TypeVar('_T') +if T.TYPE_CHECKING: + from .environment import Environment + # These classes contains all the data pulled from configuration files (native # and cross file currently), and also assists with the reading environment @@ -122,10 +125,10 @@ ENV_VAR_PROG_MAP: T.Mapping[str, str] = { # Deprecated environment variables mapped from the new variable to the old one # Deprecated in 0.54.0 DEPRECATED_ENV_PROG_MAP: T.Mapping[str, str] = { - 'DC_LD': 'D_LD', - 'FC_LD': 'F_LD', - 'RUSTC_LD': 'RUST_LD', - 'OBJCXX_LD': 'OBJCPP_LD', + 'd_ld': 'D_LD', + 'fortran_ld': 'F_LD', + 'rust_ld': 'RUST_LD', + 'objcpp_ld': 'OBJCPP_LD', } class CMakeSkipCompilerTest(Enum): @@ -394,17 +397,18 @@ class MachineInfo: return self.is_windows() or self.is_cygwin() class BinaryTable: + def __init__( self, binaries: T.Optional[T.Dict[str, T.Union[str, T.List[str]]]] = None, ): - self.binaries = binaries or {} # type: T.Dict[str, T.Union[str, T.List[str]]] - for name, command in self.binaries.items(): - if not isinstance(command, (list, str)): - # TODO generalize message - raise mesonlib.MesonException( - 'Invalid type {!r} for binary {!r} in cross file' - ''.format(command, name)) + self.binaries: T.Dict[str, T.List[str]] = {} + if binaries: + for name, command in binaries.items(): + if not isinstance(command, (list, str)): + raise mesonlib.MesonException( + f'Invalid type {command!r} for entry {name!r} in cross file') + self.binaries[name] = mesonlib.listify(command) @staticmethod def detect_ccache() -> T.List[str]: @@ -426,42 +430,17 @@ class BinaryTable: # Return value has to be a list of compiler 'choices' return compiler, ccache - def lookup_entry(self, - for_machine: MachineChoice, - is_cross: bool, - name: str) -> T.Optional[T.List[str]]: + def lookup_entry(self, name: str) -> T.Optional[T.List[str]]: """Lookup binary in cross/native file and fallback to environment. Returns command with args as list if found, Returns `None` if nothing is found. """ - # Try explicit map, don't fall back on env var - # Try explict map, then env vars - for _ in [()]: # a trick to get `break` - raw_command = self.binaries.get(name) - if raw_command is not None: - command = mesonlib.stringlistify(raw_command) - break # found - evar = ENV_VAR_PROG_MAP.get(name) - if evar is not None: - raw_command = get_env_var(for_machine, is_cross, evar) - if raw_command is None: - deprecated = DEPRECATED_ENV_PROG_MAP.get(evar) - if deprecated is not None: - raw_command = get_env_var(for_machine, is_cross, deprecated) - if raw_command is not None: - mlog.deprecation( - 'The', deprecated, 'environment variable is deprecated in favor of', - evar, once=True) - if raw_command is not None: - command = split_args(raw_command) - break # found - command = None - - - # Do not return empty or blank string entries - if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): - command = None + command = self.binaries.get(name) + if not command: + return None + elif not command[0].strip(): + return None return command class CMakeVariables: diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 12b9dbb49..6275da05a 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -29,7 +29,7 @@ from .mesonlib import ( from . import mlog from .envconfig import ( - BinaryTable, MachineInfo, + BinaryTable, ENV_VAR_PROG_MAP, MachineInfo, Properties, known_cpu_families, get_env_var_pair, CMakeVariables, ) @@ -129,6 +129,7 @@ from .compilers import ( VisualStudioCCompiler, VisualStudioCPPCompiler, ) +from mesonbuild import envconfig if T.TYPE_CHECKING: from configparser import ConfigParser @@ -654,6 +655,7 @@ class Environment: # Take default value from env if not set in cross/native files or command line. self.set_default_options_from_env() + self._set_default_binaries_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other @@ -804,6 +806,21 @@ class Environment: v = mesonlib.listify(self.options.get(key, [])) self.options.setdefault(key, v + p_list) + def _set_default_binaries_from_env(self) -> None: + """Set default binaries from the environment. + + For example, pkg-config can be set via PKG_CONFIG, or in the machine + file. We want to set the default to the env variable. + """ + opts = itertools.chain(envconfig.DEPRECATED_ENV_PROG_MAP.items(), + envconfig.ENV_VAR_PROG_MAP.items()) + + for (name, evar), for_machine in itertools.product(opts, MachineChoice): + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) + def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets # re-initialized with project options by the interpreter during @@ -853,11 +870,8 @@ class Environment: def is_library(self, fname): return is_library(fname) - def lookup_binary_entry(self, for_machine: MachineChoice, name: str): - return self.binaries[for_machine].lookup_entry( - for_machine, - self.is_cross_build(), - name) + def lookup_binary_entry(self, for_machine: MachineChoice, name: str) -> T.List[str]: + return self.binaries[for_machine].lookup_entry(name) @staticmethod def get_gnu_compiler_defines(compiler): diff --git a/run_unittests.py b/run_unittests.py index cf78e3ef2..bbcdb5fb7 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -2637,17 +2637,16 @@ class AllPlatformTests(BasePlatformTests): # something like `ccache gcc -pipe` or `distcc ccache gcc` works. wrapper = os.path.join(testdir, 'compiler wrapper.py') wrappercc = python_command + [wrapper] + cc.get_exelist() + ['-DSOME_ARG'] - wrappercc_s = '' - for w in wrappercc: - wrappercc_s += quote_arg(w) + ' ' - os.environ[evar] = wrappercc_s - wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) + os.environ[evar] = ' '.join(quote_arg(w) for w in wrappercc) + # Check static linker too wrapperlinker = python_command + [wrapper] + linker.get_exelist() + linker.get_always_args() - wrapperlinker_s = '' - for w in wrapperlinker: - wrapperlinker_s += quote_arg(w) + ' ' - os.environ['AR'] = wrapperlinker_s + os.environ['AR'] = ' '.join(quote_arg(w) for w in wrapperlinker) + + # Need a new env to re-run environment loading + env = get_fake_env(testdir, self.builddir, self.prefix) + + wcc = getattr(env, 'detect_{}_compiler'.format(lang))(MachineChoice.HOST) wlinker = env.detect_static_linker(wcc) # Pop it so we don't use it for the next detection evalue = os.environ.pop('AR') @@ -5692,9 +5691,9 @@ class WindowsTests(BasePlatformTests): envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -7293,12 +7292,12 @@ class LinuxlikeTests(BasePlatformTests): raise unittest.SkipTest('Solaris currently cannot override the linker.') if not shutil.which(check): raise unittest.SkipTest('Could not find {}.'.format(check)) - envvars = [mesonbuild.envconfig.BinaryTable.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] + envvars = [mesonbuild.envconfig.ENV_VAR_PROG_MAP['{}_ld'.format(lang)]] # Also test a deprecated variable if there is one. - if envvars[0] in mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP: + if f'{lang}_ld' in mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP: envvars.append( - mesonbuild.envconfig.BinaryTable.DEPRECATION_MAP[envvars[0]]) + mesonbuild.envconfig.DEPRECATED_ENV_PROG_MAP[f'{lang}_ld']) for envvar in envvars: with mock.patch.dict(os.environ, {envvar: name}): @@ -8274,7 +8273,7 @@ class NativeFileTests(BasePlatformTests): return 'gfortran', 'gcc' self.helper_for_compiler('fortran', cb) - def _single_implementation_compiler(self, lang, binary, version_str, version): + def _single_implementation_compiler(self, lang: str, binary: str, version_str: str, version: str) -> None: """Helper for languages with a single (supported) implementation. Builds a wrapper around the compiler to override the version. @@ -8283,7 +8282,7 @@ class NativeFileTests(BasePlatformTests): env = get_fake_env() getter = getattr(env, 'detect_{}_compiler'.format(lang)) getter = functools.partial(getter, MachineChoice.HOST) - env.binaries.host.binaries[lang] = wrapper + env.binaries.host.binaries[lang] = [wrapper] compiler = getter() self.assertEqual(compiler.version, version) @@ -8310,7 +8309,7 @@ class NativeFileTests(BasePlatformTests): 'swiftc', version='Swift 1.2345', outfile='stderr', extra_args={'Xlinker': 'macosx_version. PROJECT:ld - 1.2.3'}) env = get_fake_env() - env.binaries.host.binaries['swift'] = wrapper + env.binaries.host.binaries['swift'] = [wrapper] compiler = env.detect_swift_compiler(MachineChoice.HOST) self.assertEqual(compiler.version, '1.2345') From f3fcbba1f89f0b2a323dd596a3caf4fce5a1611e Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:27:26 -0800 Subject: [PATCH 11/14] boost: default machine file properties to env var values This both moves the env reading to configuration time, which is useful, and also simplifies the implementation of the boost dependency. The simplification comes from being able to delete basically duplicated code since the values will be in the Properties if they exist at all. --- mesonbuild/dependencies/boost.py | 65 ++++++++------------------------ mesonbuild/envconfig.py | 8 ---- mesonbuild/environment.py | 21 +++++++++++ run_unittests.py | 2 +- 4 files changed, 37 insertions(+), 59 deletions(-) diff --git a/mesonbuild/dependencies/boost.py b/mesonbuild/dependencies/boost.py index 622ee3774..d12f37c3a 100644 --- a/mesonbuild/dependencies/boost.py +++ b/mesonbuild/dependencies/boost.py @@ -20,12 +20,14 @@ from pathlib import Path from .. import mlog from .. import mesonlib -from ..envconfig import get_env_var from ..environment import Environment from .base import DependencyException, ExternalDependency, PkgConfigDependency from .misc import threads_factory +if T.TYPE_CHECKING: + from ..environment import Properties + # On windows 3 directory layouts are supported: # * The default layout (versioned) installed: # - $BOOST_ROOT/include/boost-x_x/boost/*.hpp @@ -372,18 +374,11 @@ class BoostDependency(ExternalDependency): # First, look for paths specified in a machine file props = self.env.properties[self.for_machine] - boost_property_env = [props.get('boost_includedir'), props.get('boost_librarydir'), props.get('boost_root')] - if any(boost_property_env): + if any(x in self.env.properties[self.for_machine] for x in + ['boost_includedir', 'boost_librarydir', 'boost_root']): self.detect_boost_machine_file(props) return - # Next, look for paths in the environment - boost_manual_env_list = ['BOOST_INCLUDEDIR', 'BOOST_LIBRARYDIR', 'BOOST_ROOT', 'BOOSTROOT'] - boost_manual_env = [get_env_var(self.for_machine, self.env.is_cross_build, x) for x in boost_manual_env_list] - if any(boost_manual_env): - self.detect_boost_env() - return - # Finally, look for paths from .pc files and from searching the filesystem self.detect_roots() @@ -405,13 +400,20 @@ class BoostDependency(ExternalDependency): self.boost_root = j break - def detect_boost_machine_file(self, props: T.Dict[str, str]) -> None: + def detect_boost_machine_file(self, props: 'Properties') -> None: + """Detect boost with values in the machine file or environment. + + The machine file values are defaulted to the environment values. + """ + # XXX: if we had a TypedDict we woudn't need this incdir = props.get('boost_includedir') + assert incdir is None or isinstance(incdir, str) libdir = props.get('boost_librarydir') + assert libdir is None or isinstance(libdir, str) if incdir and libdir: - inc_dir = Path(props['boost_includedir']) - lib_dir = Path(props['boost_librarydir']) + inc_dir = Path(incdir) + lib_dir = Path(libdir) if not inc_dir.is_absolute() or not lib_dir.is_absolute(): raise DependencyException('Paths given for boost_includedir and boost_librarydir in machine file must be absolute') @@ -436,43 +438,6 @@ class BoostDependency(ExternalDependency): self.check_and_set_roots(paths) - def detect_boost_env(self) -> None: - boost_includedir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_INCLUDEDIR') - boost_librarydir = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_LIBRARYDIR') - - boost_manual_env = [boost_includedir, boost_librarydir] - if all(boost_manual_env): - inc_dir = Path(boost_includedir) - lib_dir = Path(boost_librarydir) - - if not inc_dir.is_absolute() or not lib_dir.is_absolute(): - raise DependencyException('Paths given in BOOST_INCLUDEDIR and BOOST_LIBRARYDIR must be absolute') - - mlog.debug('Trying to find boost with:') - mlog.debug(' - BOOST_INCLUDEDIR = {}'.format(inc_dir)) - mlog.debug(' - BOOST_LIBRARYDIR = {}'.format(lib_dir)) - - return self.detect_split_root(inc_dir, lib_dir) - - elif any(boost_manual_env): - raise DependencyException('Both BOOST_INCLUDEDIR *and* BOOST_LIBRARYDIR have to be set (one is not enough). Ignoring.') - - boost_root = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOST_ROOT') - boostroot = get_env_var(self.for_machine, self.env.is_cross_build, 'BOOSTROOT') - - # It shouldn't be possible to get here without something in BOOST_ROOT or BOOSTROOT - assert(boost_root or boostroot) - - for path, name in [(boost_root, 'BOOST_ROOT'), (boostroot, 'BOOSTROOT')]: - if path: - raw_paths = path.split(os.pathsep) - paths = [Path(x) for x in raw_paths] - if paths and any([not x.is_absolute() for x in paths]): - raise DependencyException('Paths in {} must be absolute'.format(name)) - break - - self.check_and_set_roots(paths) - def run_check(self, inc_dirs: T.List[BoostIncludeDir], lib_dirs: T.List[Path]) -> bool: mlog.debug(' - potential library dirs: {}'.format([x.as_posix() for x in lib_dirs])) mlog.debug(' - potential include dirs: {}'.format([x.path.as_posix() for x in inc_dirs])) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 69b667869..596defe53 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -163,14 +163,6 @@ def get_env_var_pair(for_machine: MachineChoice, mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) return var, value -def get_env_var(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[str]: - ret = get_env_var_pair(for_machine, is_cross, var_name) - if ret is None: - return None - return ret[1] - class Properties: def __init__( self, diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 6275da05a..ba24411eb 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -656,6 +656,7 @@ class Environment: # Take default value from env if not set in cross/native files or command line. self.set_default_options_from_env() self._set_default_binaries_from_env() + self._set_default_properties_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other @@ -821,6 +822,26 @@ class Environment: _, p_env = p_env_pair self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) + def _set_default_properties_from_env(self) -> None: + """Properties which can alkso be set from the environment.""" + # name, evar, split + opts: T.List[T.Tuple[str, T.List[str], bool]] = [ + ('boost_includedir', ['BOOST_INCLUDEDIR'], False), + ('boost_librarydir', ['BOOST_LIBRARYDIR'], False), + ('boost_root', ['BOOST_ROOT', 'BOOSTROOT'], True), + ] + + for (name, evars, split), for_machine in itertools.product(opts, MachineChoice): + for evar in evars: + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + if split: + self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep)) + else: + self.properties[for_machine].properties.setdefault(name, p_env) + break + def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets # re-initialized with project options by the interpreter during diff --git a/run_unittests.py b/run_unittests.py index bbcdb5fb7..21b6608a6 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -5376,7 +5376,7 @@ class FailureTests(BasePlatformTests): def test_boost_BOOST_ROOT_dependency(self): # Test BOOST_ROOT; can be run even if Boost is found or not self.assertMesonRaises("dependency('boost')", - "(BOOST_ROOT.*absolute|{})".format(self.dnf), + "(boost_root.*absolute|{})".format(self.dnf), override_envvars = {'BOOST_ROOT': 'relative/path'}) def test_dependency_invalid_method(self): From 0076db6ff9309a34e2b86a6ae3fd85ff34b8f9f3 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:37:11 -0800 Subject: [PATCH 12/14] move get_env_var_pair to environment This is only used in environment, so it should live there too. --- mesonbuild/envconfig.py | 36 ++---------------------------------- mesonbuild/environment.py | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 596defe53..c133ec92c 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -12,20 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, subprocess +import subprocess import typing as T from enum import Enum from . import mesonlib -from .mesonlib import EnvironmentException, MachineChoice, PerMachine, split_args +from .mesonlib import EnvironmentException from . import mlog from pathlib import Path -_T = T.TypeVar('_T') - -if T.TYPE_CHECKING: - from .environment import Environment - # These classes contains all the data pulled from configuration files (native # and cross file currently), and also assists with the reading environment @@ -136,33 +131,6 @@ class CMakeSkipCompilerTest(Enum): NEVER = 'never' DEP_ONLY = 'dep_only' - -def get_env_var_pair(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[T.Tuple[str, str]]: - """ - Returns the exact env var and the value. - """ - candidates = PerMachine( - # The prefixed build version takes priority, but if we are native - # compiling we fall back on the unprefixed host version. This - # allows native builds to never need to worry about the 'BUILD_*' - # ones. - ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), - # Always just the unprefixed host verions - [var_name] - )[for_machine] - for var in candidates: - value = os.environ.get(var) - if value is not None: - break - else: - formatted = ', '.join(['{!r}'.format(var) for var in candidates]) - mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) - return None - mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) - return var, value - class Properties: def __init__( self, diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index ba24411eb..8ff9574db 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -23,15 +23,13 @@ from . import coredata from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, CompCertLinker, C2000Linker, IntelVisualStudioLinker, AIXArLinker from . import mesonlib from .mesonlib import ( - MesonException, EnvironmentException, MachineChoice, Popen_safe, + MesonException, EnvironmentException, MachineChoice, Popen_safe, PerMachine, PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey ) from . import mlog from .envconfig import ( - BinaryTable, ENV_VAR_PROG_MAP, MachineInfo, - Properties, known_cpu_families, get_env_var_pair, - CMakeVariables, + BinaryTable, MachineInfo, Properties, known_cpu_families, CMakeVariables, ) from . import compilers from .compilers import ( @@ -143,6 +141,34 @@ CompilersDict = T.Dict[str, Compiler] if T.TYPE_CHECKING: import argparse + +def get_env_var_pair(for_machine: MachineChoice, + is_cross: bool, + var_name: str) -> T.Optional[T.Tuple[str, str]]: + """ + Returns the exact env var and the value. + """ + candidates = PerMachine( + # The prefixed build version takes priority, but if we are native + # compiling we fall back on the unprefixed host version. This + # allows native builds to never need to worry about the 'BUILD_*' + # ones. + ([var_name + '_FOR_BUILD'] if is_cross else [var_name]), + # Always just the unprefixed host verions + [var_name] + )[for_machine] + for var in candidates: + value = os.environ.get(var) + if value is not None: + break + else: + formatted = ', '.join(['{!r}'.format(var) for var in candidates]) + mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) + return None + mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) + return var, value + + def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): gcovr_exe = 'gcovr' try: From 1cc7e4c84cc2b411b7adbc58d7a302fa7117c6e0 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:39:14 -0800 Subject: [PATCH 13/14] clean up get_env_var_pair This function returns both the name and the value, but we never actually use the name, just the value. Also make this module private. We really want to keep all environment variable reading in the Environment class so it's done once up front. This should help with that goal. --- mesonbuild/environment.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 8ff9574db..788991177 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -142,9 +142,7 @@ if T.TYPE_CHECKING: import argparse -def get_env_var_pair(for_machine: MachineChoice, - is_cross: bool, - var_name: str) -> T.Optional[T.Tuple[str, str]]: +def _get_env_var(for_machine: MachineChoice, is_cross: bool, var_name: str) -> T.Optional[str]: """ Returns the exact env var and the value. """ @@ -166,7 +164,7 @@ def get_env_var_pair(for_machine: MachineChoice, mlog.debug('None of {} are defined in the environment, not changing global flags.'.format(formatted)) return None mlog.debug('Using {!r} from environment with value: {!r}'.format(var, value)) - return var, value + return value def detect_gcovr(min_version='3.3', new_rootdir_version='4.2', log=False): @@ -791,10 +789,8 @@ class Environment: ) for (evar, keyname), for_machine in itertools.product(opts, MachineChoice): - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair - + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: # these may contain duplicates, which must be removed, else # a duplicates-in-array-option warning arises. if keyname == 'cmake_prefix_path': @@ -843,9 +839,8 @@ class Environment: envconfig.ENV_VAR_PROG_MAP.items()) for (name, evar), for_machine in itertools.product(opts, MachineChoice): - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: self.binaries[for_machine].binaries.setdefault(name, mesonlib.split_args(p_env)) def _set_default_properties_from_env(self) -> None: @@ -859,9 +854,8 @@ class Environment: for (name, evars, split), for_machine in itertools.product(opts, MachineChoice): for evar in evars: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) - if p_env_pair is not None: - _, p_env = p_env_pair + p_env = _get_env_var(for_machine, self.is_cross_build(), evar) + if p_env is not None: if split: self.properties[for_machine].properties.setdefault(name, p_env.split(os.pathsep)) else: From ff40ca25b776392625275bd7891701e02675e2b7 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 11 Dec 2020 13:40:29 -0800 Subject: [PATCH 14/14] make some Environment methods protected they're really not public methods, they'r only meant to be called from the initializer. Let's mark them as such. --- mesonbuild/environment.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 788991177..1a7f3f1c8 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -640,7 +640,7 @@ class Environment: binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) cmakevars.build = CMakeVariables(config.get('cmake', {})) - self.load_machine_file_options(config, properties.build, MachineChoice.BUILD) + self._load_machine_file_options(config, properties.build, MachineChoice.BUILD) ## Read in cross file(s) to override host machine configuration @@ -658,7 +658,7 @@ class Environment: for key, value in list(self.options.items()): if self.coredata.is_per_machine_option(key): self.options[key.as_build()] = value - self.load_machine_file_options(config, properties.host, MachineChoice.HOST) + self._load_machine_file_options(config, properties.host, MachineChoice.HOST) else: # IF we aren't cross compiling, but we hav ea native file, the # native file is for the host. This is due to an mismatch between @@ -678,7 +678,7 @@ class Environment: self.options.update(options.cmd_line_options) # Take default value from env if not set in cross/native files or command line. - self.set_default_options_from_env() + self._set_default_options_from_env() self._set_default_binaries_from_env() self._set_default_properties_from_env() @@ -746,7 +746,7 @@ class Environment: self.default_pkgconfig = ['pkg-config'] self.wrap_resolver = None - def load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: + def _load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: """Read the contents of a Machine file and put it in the options store.""" paths = config.get('paths') if paths: @@ -777,7 +777,7 @@ class Environment: key = OptionKey.from_string(k).evolve(subproject=subproject) self.options[key] = v - def set_default_options_from_env(self) -> None: + def _set_default_options_from_env(self) -> None: opts: T.List[T.Tuple[str, str]] = ( [(v, f'{k}_args') for k, v in compilers.compilers.CFLAGS_MAPPING.items()] + [