From 7e58f33376119b53e01616139ad9354ce9cfe003 Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 5 Oct 2020 20:44:27 +0200 Subject: [PATCH 1/3] cmake: Add cross compilation support --- mesonbuild/cmake/__init__.py | 7 +- mesonbuild/cmake/client.py | 14 +-- mesonbuild/cmake/common.py | 12 ++ mesonbuild/cmake/executor.py | 185 ++------------------------- mesonbuild/cmake/interpreter.py | 100 +++++++-------- mesonbuild/cmake/toolchain.py | 217 ++++++++++++++++++++++++++++++++ mesonbuild/dependencies/base.py | 30 +++-- mesonbuild/envconfig.py | 97 ++++++++++++-- mesonbuild/environment.py | 13 +- run_unittests.py | 1 + 10 files changed, 408 insertions(+), 268 deletions(-) create mode 100644 mesonbuild/cmake/toolchain.py diff --git a/mesonbuild/cmake/__init__.py b/mesonbuild/cmake/__init__.py index db7aefd6f..9840a5f4a 100644 --- a/mesonbuild/cmake/__init__.py +++ b/mesonbuild/cmake/__init__.py @@ -18,10 +18,12 @@ __all__ = [ 'CMakeClient', 'CMakeExecutor', + 'CMakeExecScope', 'CMakeException', 'CMakeFileAPI', 'CMakeInterpreter', 'CMakeTarget', + 'CMakeToolchain', 'CMakeTraceLine', 'CMakeTraceParser', 'SingleTargetOptions', @@ -31,10 +33,11 @@ __all__ = [ 'cmake_defines_to_args', ] -from .common import CMakeException, SingleTargetOptions, TargetOptions, cmake_defines_to_args +from .common import CMakeException, SingleTargetOptions, TargetOptions, cmake_defines_to_args, language_map from .client import CMakeClient from .executor import CMakeExecutor from .fileapi import CMakeFileAPI from .generator import parse_generator_expressions -from .interpreter import CMakeInterpreter, language_map +from .interpreter import CMakeInterpreter +from .toolchain import CMakeToolchain, CMakeExecScope from .traceparser import CMakeTarget, CMakeTraceLine, CMakeTraceParser diff --git a/mesonbuild/cmake/client.py b/mesonbuild/cmake/client.py index ce79e8ef9..8f9456b56 100644 --- a/mesonbuild/cmake/client.py +++ b/mesonbuild/cmake/client.py @@ -16,8 +16,6 @@ # or an interpreter-based tool. from .common import CMakeException, CMakeConfiguration, CMakeBuildFile -from .executor import CMakeExecutor -from ..mesonlib import MachineChoice from .. import mlog from contextlib import contextmanager from subprocess import Popen, PIPE, TimeoutExpired @@ -27,6 +25,7 @@ import json if T.TYPE_CHECKING: from ..environment import Environment + from .executor import CMakeExecutor CMAKE_SERVER_BEGIN_STR = '[== "CMake Server" ==[' CMAKE_SERVER_END_STR = ']== "CMake Server" ==]' @@ -331,20 +330,17 @@ class CMakeClient: return ReplyCMakeInputs(data['cookie'], Path(data['cmakeRootDirectory']), Path(data['sourceDirectory']), files) @contextmanager - def connect(self) -> T.Generator[None, None, None]: - self.startup() + def connect(self, cmake_exe: 'CMakeExecutor') -> T.Generator[None, None, None]: + self.startup(cmake_exe) try: yield finally: self.shutdown() - def startup(self) -> None: + def startup(self, cmake_exe: 'CMakeExecutor') -> None: if self.proc is not None: raise CMakeException('The CMake server was already started') - for_machine = MachineChoice.HOST # TODO make parameter - cmake_exe = CMakeExecutor(self.env, '>=3.7', for_machine) - if not cmake_exe.found(): - raise CMakeException('Unable to find CMake') + assert cmake_exe.found() mlog.debug('Starting CMake server with CMake', mlog.bold(' '.join(cmake_exe.get_command())), 'version', mlog.cyan(cmake_exe.version())) self.proc = Popen(cmake_exe.get_command() + ['-E', 'server', '--experimental', '--debug'], stdin=PIPE, stdout=PIPE) diff --git a/mesonbuild/cmake/common.py b/mesonbuild/cmake/common.py index 21460caa6..3534ec00a 100644 --- a/mesonbuild/cmake/common.py +++ b/mesonbuild/cmake/common.py @@ -20,6 +20,18 @@ from .. import mlog from .._pathlib import Path import typing as T +language_map = { + 'c': 'C', + 'cpp': 'CXX', + 'cuda': 'CUDA', + 'objc': 'OBJC', + 'objcpp': 'OBJCXX', + 'cs': 'CSharp', + 'java': 'Java', + 'fortran': 'Fortran', + 'swift': 'Swift', +} + class CMakeException(MesonException): pass diff --git a/mesonbuild/cmake/executor.py b/mesonbuild/cmake/executor.py index 0413b56b2..e11dbe996 100644 --- a/mesonbuild/cmake/executor.py +++ b/mesonbuild/cmake/executor.py @@ -21,59 +21,19 @@ from threading import Thread import typing as T import re import os -import shutil -import ctypes -import textwrap -from .. import mlog, mesonlib -from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice -from ..environment import Environment +from .. import mlog +from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows from ..envconfig import get_env_var -from ..compilers import ( - AppleClangCCompiler, AppleClangCPPCompiler, AppleClangObjCCompiler, - AppleClangObjCPPCompiler -) if T.TYPE_CHECKING: + from ..environment import Environment from ..dependencies.base import ExternalProgram from ..compilers import Compiler TYPE_result = T.Tuple[int, T.Optional[str], T.Optional[str]] TYPE_cache_key = T.Tuple[str, T.Tuple[str, ...], str, T.FrozenSet[T.Tuple[str, str]]] -_MESON_TO_CMAKE_MAPPING = { - 'arm': 'ARMCC', - 'armclang': 'ARMClang', - 'clang': 'Clang', - 'clang-cl': 'MSVC', - 'flang': 'Flang', - 'g95': 'G95', - 'gcc': 'GNU', - 'intel': 'Intel', - 'intel-cl': 'MSVC', - 'msvc': 'MSVC', - 'pathscale': 'PathScale', - 'pgi': 'PGI', - 'sun': 'SunPro', -} - -def meson_compiler_to_cmake_id(cobj: 'Compiler') -> str: - """Translate meson compiler's into CMAKE compiler ID's. - - Most of these can be handled by a simple table lookup, with a few - exceptions. - - Clang and Apple's Clang are both identified as "clang" by meson. To make - things more complicated gcc and vanilla clang both use Apple's ld64 on - macOS. The only way to know for sure is to do an isinstance() check. - """ - if isinstance(cobj, (AppleClangCCompiler, AppleClangCPPCompiler, - AppleClangObjCCompiler, AppleClangObjCPPCompiler)): - return 'AppleClang' - # If no mapping, try GNU and hope that the build files don't care - return _MESON_TO_CMAKE_MAPPING.get(cobj.get_id(), 'GNU') - - class CMakeExecutor: # The class's copy of the CMake path. Avoids having to search for it # multiple times in the same Meson invocation. @@ -81,7 +41,7 @@ class CMakeExecutor: class_cmakevers = PerMachine(None, None) # type: PerMachine[T.Optional[str]] class_cmake_cache = {} # type: T.Dict[T.Any, TYPE_result] - def __init__(self, environment: Environment, version: str, for_machine: MachineChoice, silent: bool = False): + def __init__(self, environment: 'Environment', version: str, for_machine: MachineChoice, silent: bool = False): self.min_version = version self.environment = environment self.for_machine = for_machine @@ -109,7 +69,7 @@ class CMakeExecutor: 'CMAKE_PREFIX_PATH') if env_pref_path_raw is not None: env_pref_path = [] # type: T.List[str] - if mesonlib.is_windows(): + if is_windows(): # Cannot split on ':' on Windows because its in the drive letter env_pref_path = env_pref_path_raw.split(os.pathsep) else: @@ -123,7 +83,7 @@ class CMakeExecutor: if self.prefix_paths: self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))] - def find_cmake_binary(self, environment: Environment, silent: bool = False) -> T.Tuple[T.Optional['ExternalProgram'], T.Optional[str]]: + def find_cmake_binary(self, environment: 'Environment', silent: bool = False) -> T.Tuple[T.Optional['ExternalProgram'], T.Optional[str]]: from ..dependencies.base import find_external_program, NonExistingExternalProgram # Only search for CMake the first time and store the result in the class @@ -176,7 +136,7 @@ class CMakeExecutor: return None except PermissionError: msg = 'Found CMake {!r} but didn\'t have permissions to run it.'.format(' '.join(cmakebin.get_command())) - if not mesonlib.is_windows(): + if not is_windows(): msg += '\n\nOn Unix-like systems this is often caused by scripts that are not executable.' mlog.warning(msg) return None @@ -257,11 +217,12 @@ class CMakeExecutor: rc = ret.returncode out = ret.stdout.decode(errors='ignore') err = ret.stderr.decode(errors='ignore') - call = ' '.join(cmd) - mlog.debug("Called `{}` in {} -> {}".format(call, build_dir, rc)) return rc, out, err def _call_impl(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result: + mlog.debug('Calling CMake ({}) in {} with:'.format(self.cmakebin.get_command(), build_dir)) + for i in args: + mlog.debug(' - "{}"'.format(i)) if not self.print_cmout: return self._call_quiet(args, build_dir, env) else: @@ -285,132 +246,6 @@ class CMakeExecutor: cache[key] = self._call_impl(args, build_dir, env) return cache[key] - def call_with_fake_build(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]] = None) -> TYPE_result: - # First check the cache - cache = CMakeExecutor.class_cmake_cache - key = self._cache_key(args, build_dir, env) - if key in cache: - return cache[key] - - build_dir.mkdir(exist_ok=True, parents=True) - - # Try to set the correct compiler for C and C++ - # This step is required to make try_compile work inside CMake - fallback = Path(__file__).resolve() # A file used as a fallback wehen everything else fails - compilers = self.environment.coredata.compilers[MachineChoice.BUILD] - - def make_abs(exe: str, lang: str) -> str: - if Path(exe).is_absolute(): - return exe - - p = shutil.which(exe) - if p is None: - mlog.debug('Failed to find a {} compiler for CMake. This might cause CMake to fail.'.format(lang)) - return str(fallback) - return p - - def choose_compiler(lang: str) -> T.Tuple[str, str, str, str]: - comp_obj = None - exe_list = [] - if lang in compilers: - comp_obj = compilers[lang] - else: - try: - comp_obj = self.environment.compiler_from_language(lang, MachineChoice.BUILD) - except Exception: - pass - - if comp_obj is not None: - exe_list = comp_obj.get_exelist() - comp_id = meson_compiler_to_cmake_id(comp_obj) - comp_version = comp_obj.version.upper() - - if len(exe_list) == 1: - return make_abs(exe_list[0], lang), '', comp_id, comp_version - elif len(exe_list) == 2: - return make_abs(exe_list[1], lang), make_abs(exe_list[0], lang), comp_id, comp_version - else: - mlog.debug('Failed to find a {} compiler for CMake. This might cause CMake to fail.'.format(lang)) - return str(fallback), '', 'GNU', '' - - c_comp, c_launcher, c_id, c_version = choose_compiler('c') - cxx_comp, cxx_launcher, cxx_id, cxx_version = choose_compiler('cpp') - fortran_comp, fortran_launcher, _, _ = choose_compiler('fortran') - - # on Windows, choose_compiler returns path with \ as separator - replace by / before writing to CMAKE file - c_comp = c_comp.replace('\\', '/') - c_launcher = c_launcher.replace('\\', '/') - cxx_comp = cxx_comp.replace('\\', '/') - cxx_launcher = cxx_launcher.replace('\\', '/') - fortran_comp = fortran_comp.replace('\\', '/') - fortran_launcher = fortran_launcher.replace('\\', '/') - - # Reset the CMake cache - (build_dir / 'CMakeCache.txt').write_text('CMAKE_PLATFORM_INFO_INITIALIZED:INTERNAL=1\n') - - # Fake the compiler files - comp_dir = build_dir / 'CMakeFiles' / self.cmakevers - comp_dir.mkdir(parents=True, exist_ok=True) - - c_comp_file = comp_dir / 'CMakeCCompiler.cmake' - cxx_comp_file = comp_dir / 'CMakeCXXCompiler.cmake' - fortran_comp_file = comp_dir / 'CMakeFortranCompiler.cmake' - - if c_comp and not c_comp_file.is_file(): - is_gnu = '1' if c_id == 'GNU' else '' - c_comp_file.write_text(textwrap.dedent('''\ - # Fake CMake file to skip the boring and slow stuff - set(CMAKE_C_COMPILER "{}") # Should be a valid compiler for try_compile, etc. - set(CMAKE_C_COMPILER_LAUNCHER "{}") # The compiler launcher (if presentt) - set(CMAKE_COMPILER_IS_GNUCC {}) - set(CMAKE_C_COMPILER_ID "{}") - set(CMAKE_C_COMPILER_VERSION "{}") - set(CMAKE_C_COMPILER_LOADED 1) - set(CMAKE_C_COMPILER_FORCED 1) - set(CMAKE_C_COMPILER_WORKS TRUE) - set(CMAKE_C_ABI_COMPILED TRUE) - set(CMAKE_C_SOURCE_FILE_EXTENSIONS c;m) - set(CMAKE_C_IGNORE_EXTENSIONS h;H;o;O;obj;OBJ;def;DEF;rc;RC) - set(CMAKE_SIZEOF_VOID_P "{}") - '''.format(c_comp, c_launcher, is_gnu, c_id, c_version, - ctypes.sizeof(ctypes.c_void_p)))) - - if cxx_comp and not cxx_comp_file.is_file(): - is_gnu = '1' if cxx_id == 'GNU' else '' - cxx_comp_file.write_text(textwrap.dedent('''\ - # Fake CMake file to skip the boring and slow stuff - set(CMAKE_CXX_COMPILER "{}") # Should be a valid compiler for try_compile, etc. - set(CMAKE_CXX_COMPILER_LAUNCHER "{}") # The compiler launcher (if presentt) - set(CMAKE_COMPILER_IS_GNUCXX {}) - set(CMAKE_CXX_COMPILER_ID "{}") - set(CMAKE_CXX_COMPILER_VERSION "{}") - set(CMAKE_CXX_COMPILER_LOADED 1) - set(CMAKE_CXX_COMPILER_FORCED 1) - set(CMAKE_CXX_COMPILER_WORKS TRUE) - set(CMAKE_CXX_ABI_COMPILED TRUE) - set(CMAKE_CXX_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC) - set(CMAKE_CXX_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;mm;CPP) - set(CMAKE_SIZEOF_VOID_P "{}") - '''.format(cxx_comp, cxx_launcher, is_gnu, cxx_id, cxx_version, - ctypes.sizeof(ctypes.c_void_p)))) - - if fortran_comp and not fortran_comp_file.is_file(): - fortran_comp_file.write_text(textwrap.dedent('''\ - # Fake CMake file to skip the boring and slow stuff - set(CMAKE_Fortran_COMPILER "{}") # Should be a valid compiler for try_compile, etc. - set(CMAKE_Fortran_COMPILER_LAUNCHER "{}") # The compiler launcher (if presentt) - set(CMAKE_Fortran_COMPILER_ID "GNU") # Pretend we have found GCC - set(CMAKE_COMPILER_IS_GNUG77 1) - set(CMAKE_Fortran_COMPILER_LOADED 1) - set(CMAKE_Fortran_COMPILER_WORKS TRUE) - set(CMAKE_Fortran_ABI_COMPILED TRUE) - set(CMAKE_Fortran_IGNORE_EXTENSIONS h;H;o;O;obj;OBJ;def;DEF;rc;RC) - set(CMAKE_Fortran_SOURCE_FILE_EXTENSIONS f;F;fpp;FPP;f77;F77;f90;F90;for;For;FOR;f95;F95) - set(CMAKE_SIZEOF_VOID_P "{}") - '''.format(fortran_comp, fortran_launcher, ctypes.sizeof(ctypes.c_void_p)))) - - return self.call(args, build_dir, env) - def found(self) -> bool: return self.cmakebin is not None diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index cad45099f..03ed90d26 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -15,10 +15,11 @@ # This class contains the basic functionality needed to run any interpreter # or an interpreter-based tool. -from .common import CMakeException, CMakeTarget, TargetOptions, CMakeConfiguration +from .common import CMakeException, CMakeTarget, TargetOptions, CMakeConfiguration, language_map from .client import CMakeClient, RequestCMakeInputs, RequestConfigure, RequestCompute, RequestCodeModel, ReplyCMakeInputs, ReplyCodeModel from .fileapi import CMakeFileAPI from .executor import CMakeExecutor +from .toolchain import CMakeToolchain, CMakeExecScope from .traceparser import CMakeTraceParser, CMakeGeneratorTarget from .. import mlog, mesonlib from ..mesonlib import MachineChoice, OrderedSet, version_compare, path_is_in_root, relative_to_if_possible @@ -69,6 +70,7 @@ disable_policy_warnings = [ 'CMP0067', 'CMP0082', 'CMP0089', + 'CMP0102', ] backend_generator_map = { @@ -80,18 +82,6 @@ backend_generator_map = { 'vs2019': 'Visual Studio 16 2019', } -language_map = { - 'c': 'C', - 'cpp': 'CXX', - 'cuda': 'CUDA', - 'objc': 'OBJC', - 'objcpp': 'OBJCXX', - 'cs': 'CSharp', - 'java': 'Java', - 'fortran': 'Fortran', - 'swift': 'Swift', -} - target_type_map = { 'STATIC_LIBRARY': 'static_library', 'MODULE_LIBRARY': 'shared_module', @@ -221,8 +211,9 @@ class OutputTargetMap: return '__art_{}__'.format(fname.name) class ConverterTarget: - def __init__(self, target: CMakeTarget, env: 'Environment') -> None: + def __init__(self, target: CMakeTarget, env: 'Environment', for_machine: MachineChoice) -> None: self.env = env + self.for_machine = for_machine self.artifacts = target.artifacts self.src_dir = target.src_dir self.build_dir = target.build_dir @@ -653,7 +644,7 @@ class ConverterCustomTarget: tgt_counter = 0 # type: int out_counter = 0 # type: int - def __init__(self, target: CMakeGeneratorTarget) -> None: + def __init__(self, target: CMakeGeneratorTarget, env: 'Environment', for_machine: MachineChoice) -> None: assert target.current_bin_dir is not None assert target.current_src_dir is not None self.name = target.name @@ -671,6 +662,8 @@ class ConverterCustomTarget: self.depends = [] # type: T.List[T.Union[ConverterTarget, ConverterCustomTarget]] self.current_bin_dir = target.current_bin_dir # type: Path self.current_src_dir = target.current_src_dir # type: Path + self.env = env + self.for_machine = for_machine self._raw_target = target # Convert the target name to a valid meson target name @@ -723,6 +716,11 @@ class ConverterCustomTarget: continue target = output_target_map.executable(j) if target: + # When cross compiling, binaries have to be executed with an exe_wrapper (for instance wine for mingw-w64) + if self.env.exe_wrapper is not None and self.env.properties[self.for_machine].get_cmake_use_exe_wrapper(): + from ..dependencies import ExternalProgram + assert isinstance(self.env.exe_wrapper, ExternalProgram) + cmd += self.env.exe_wrapper.get_command() cmd += [target] continue elif j in trace.targets: @@ -821,6 +819,7 @@ class CMakeInterpreter: self.build_dir = Path(env.get_build_dir()) / self.build_dir_rel self.install_prefix = install_prefix self.env = env + self.for_machine = MachineChoice.HOST # TODO make parameter self.backend_name = backend.name self.linkers = set() # type: T.Set[str] self.cmake_api = CMakeAPI.SERVER @@ -844,65 +843,53 @@ class CMakeInterpreter: self.generated_targets = {} # type: T.Dict[str, T.Dict[str, T.Optional[str]]] self.internal_name_map = {} # type: T.Dict[str, str] - def configure(self, extra_cmake_options: T.List[str]) -> None: - for_machine = MachineChoice.HOST # TODO make parameter + # Do some special handling for object libraries for certain configurations + self._object_lib_workaround = False + if self.backend_name.startswith('vs'): + for comp in self.env.coredata.compilers[self.for_machine].values(): + if comp.get_linker_id() == 'link': + self._object_lib_workaround = True + break + + def configure(self, extra_cmake_options: T.List[str]) -> CMakeExecutor: # Find CMake - cmake_exe = CMakeExecutor(self.env, '>=3.7', for_machine) + cmake_exe = CMakeExecutor(self.env, '>=3.7', MachineChoice.BUILD) if not cmake_exe.found(): raise CMakeException('Unable to find CMake') self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True) preload_file = mesondata['cmake/data/preload.cmake'].write_to_private(self.env) - - # Prefere CMAKE_PROJECT_INCLUDE over CMAKE_TOOLCHAIN_FILE if possible, - # since CMAKE_PROJECT_INCLUDE was actually designed for code injection. - preload_var = 'CMAKE_PROJECT_INCLUDE' - if version_compare(cmake_exe.version(), '<3.15'): - preload_var = 'CMAKE_TOOLCHAIN_FILE' + toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.SUBPROJECT, self.build_dir.parent, preload_file) + toolchain_file = toolchain.write() generator = backend_generator_map[self.backend_name] cmake_args = [] + cmake_args += ['-G', generator] + cmake_args += ['-DCMAKE_INSTALL_PREFIX={}'.format(self.install_prefix)] + cmake_args += extra_cmake_options trace_args = self.trace.trace_args() cmcmp_args = ['-DCMAKE_POLICY_WARNING_{}=OFF'.format(x) for x in disable_policy_warnings] - pload_args = ['-D{}={}'.format(preload_var, str(preload_file))] if version_compare(cmake_exe.version(), '>=3.14'): self.cmake_api = CMakeAPI.FILE self.fileapi.setup_request() - # Map meson compiler to CMake variables - for lang, comp in self.env.coredata.compilers[for_machine].items(): - if lang not in language_map: - continue - self.linkers.add(comp.get_linker_id()) - cmake_lang = language_map[lang] - exelist = comp.get_exelist() - if len(exelist) == 1: - cmake_args += ['-DCMAKE_{}_COMPILER={}'.format(cmake_lang, exelist[0])] - elif len(exelist) == 2: - cmake_args += ['-DCMAKE_{}_COMPILER_LAUNCHER={}'.format(cmake_lang, exelist[0]), - '-DCMAKE_{}_COMPILER={}'.format(cmake_lang, exelist[1])] - if hasattr(comp, 'get_linker_exelist') and comp.get_id() == 'clang-cl': - cmake_args += ['-DCMAKE_LINKER={}'.format(comp.get_linker_exelist()[0])] - cmake_args += ['-G', generator] - cmake_args += ['-DCMAKE_INSTALL_PREFIX={}'.format(self.install_prefix)] - cmake_args += extra_cmake_options - # Run CMake mlog.log() with mlog.nested(): mlog.log('Configuring the build directory with', mlog.bold('CMake'), 'version', mlog.cyan(cmake_exe.version())) - mlog.log(mlog.bold('Running:'), ' '.join(cmake_args)) + mlog.log(mlog.bold('Running CMake with:'), ' '.join(cmake_args)) mlog.log(mlog.bold(' - build directory: '), self.build_dir.as_posix()) mlog.log(mlog.bold(' - source directory: '), self.src_dir.as_posix()) - mlog.log(mlog.bold(' - trace args: '), ' '.join(trace_args)) + mlog.log(mlog.bold(' - toolchain file: '), toolchain_file.as_posix()) mlog.log(mlog.bold(' - preload file: '), preload_file.as_posix()) + mlog.log(mlog.bold(' - trace args: '), ' '.join(trace_args)) mlog.log(mlog.bold(' - disabled policy warnings:'), '[{}]'.format(', '.join(disable_policy_warnings))) mlog.log() self.build_dir.mkdir(parents=True, exist_ok=True) os_env = environ.copy() os_env['LC_ALL'] = 'C' - final_args = cmake_args + trace_args + cmcmp_args + pload_args + [self.src_dir.as_posix()] + final_args = cmake_args + trace_args + cmcmp_args + toolchain.get_cmake_args() + [self.src_dir.as_posix()] cmake_exe.set_exec_mode(print_cmout=True, always_capture_stderr=self.trace.requires_stderr()) rc, _, self.raw_trace = cmake_exe.call(final_args, self.build_dir, env=os_env, disable_cache=True) @@ -913,11 +900,13 @@ class CMakeInterpreter: if rc != 0: raise CMakeException('Failed to configure the CMake subproject') + return cmake_exe + def initialise(self, extra_cmake_options: T.List[str]) -> None: # Run configure the old way because doing it # with the server doesn't work for some reason # Additionally, the File API requires a configure anyway - self.configure(extra_cmake_options) + cmake_exe = self.configure(extra_cmake_options) # Continue with the file API If supported if self.cmake_api is CMakeAPI.FILE: @@ -934,7 +923,7 @@ class CMakeInterpreter: self.codemodel_configs = self.fileapi.get_cmake_configurations() return - with self.client.connect(): + with self.client.connect(cmake_exe): generator = backend_generator_map[self.backend_name] self.client.do_handshake(self.src_dir, self.build_dir, generator, 1) @@ -982,7 +971,7 @@ class CMakeInterpreter: # dummy CMake internal target types if k_0.type not in skip_targets and k_0.name not in added_target_names: added_target_names += [k_0.name] - self.targets += [ConverterTarget(k_0, self.env)] + self.targets += [ConverterTarget(k_0, self.env, self.for_machine)] # Add interface targets from trace, if not already present. # This step is required because interface targets were removed from @@ -997,10 +986,10 @@ class CMakeInterpreter: 'sourceDirectory': self.src_dir, 'buildDirectory': self.build_dir, }) - self.targets += [ConverterTarget(dummy, self.env)] + self.targets += [ConverterTarget(dummy, self.env, self.for_machine)] for i_2 in self.trace.custom_targets: - self.custom_targets += [ConverterCustomTarget(i_2)] + self.custom_targets += [ConverterCustomTarget(i_2, self.env, self.for_machine)] # generate the output_target_map for i_3 in [*self.targets, *self.custom_targets]: @@ -1020,7 +1009,7 @@ class CMakeInterpreter: # Second pass: Detect object library dependencies for tgt in self.targets: - tgt.process_object_libs(object_libs, self._object_lib_workaround()) + tgt.process_object_libs(object_libs, self._object_lib_workaround) # Third pass: Reassign dependencies to avoid some loops for tgt in self.targets: @@ -1279,7 +1268,7 @@ class CMakeInterpreter: detect_cycle(tgt) tgt_var = tgt.name # type: str - def resolve_source(x: T.Any) -> T.Any: + def resolve_source(x: T.Union[str, ConverterTarget, ConverterCustomTarget, CustomTargetReference]) -> T.Union[str, IdNode, IndexNode]: if isinstance(x, ConverterTarget): if x.name not in processed: process_target(x) @@ -1296,7 +1285,7 @@ class CMakeInterpreter: return x # Generate the command list - command = [] + command = [] # type: T.List[T.Union[str, IdNode, IndexNode]] command += mesonlib.meson_command command += ['--internal', 'cmake_run_ctgt'] command += ['-o', '@OUTPUT@'] @@ -1346,6 +1335,3 @@ class CMakeInterpreter: def target_list(self) -> T.List[str]: return list(self.internal_name_map.keys()) - - def _object_lib_workaround(self) -> bool: - return 'link' in self.linkers and self.backend_name.startswith('vs') diff --git a/mesonbuild/cmake/toolchain.py b/mesonbuild/cmake/toolchain.py new file mode 100644 index 000000000..52d1f10b8 --- /dev/null +++ b/mesonbuild/cmake/toolchain.py @@ -0,0 +1,217 @@ +# Copyright 2020 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .._pathlib import Path +from ..envconfig import CMakeSkipCompilerTest +from ..mesonlib import MachineChoice +from .common import language_map +from .. import mlog + +import shutil +import typing as T +from enum import Enum +from textwrap import dedent + +if T.TYPE_CHECKING: + from ..envconfig import MachineInfo, Properties, CMakeVariables + from ..environment import Environment + from ..compilers import Compiler + + +_MESON_TO_CMAKE_MAPPING = { + 'arm': 'ARMCC', + 'armclang': 'ARMClang', + 'clang': 'Clang', + 'clang-cl': 'MSVC', + 'flang': 'Flang', + 'g95': 'G95', + 'gcc': 'GNU', + 'intel': 'Intel', + 'intel-cl': 'MSVC', + 'msvc': 'MSVC', + 'pathscale': 'PathScale', + 'pgi': 'PGI', + 'sun': 'SunPro', +} + +class CMakeExecScope(Enum): + SUBPROJECT = 'subproject' + DEPENDENCY = 'dependency' + +class CMakeToolchain: + def __init__(self, env: 'Environment', for_machine: MachineChoice, exec_scope: CMakeExecScope, out_dir: Path, preload_file: T.Optional[Path] = None) -> None: + self.env = env + self.for_machine = for_machine + self.exec_scope = exec_scope + self.preload_file = preload_file + self.toolchain_file = out_dir / 'CMakeMesonToolchainFile.cmake' + self.toolchain_file = self.toolchain_file.resolve() + self.minfo = self.env.machines[self.for_machine] + self.properties = self.env.properties[self.for_machine] + self.compilers = self.env.coredata.compilers[self.for_machine] + self.cmakevars = self.env.cmakevars[self.for_machine] + + self.variables = self.get_defaults() + self.variables.update(self.cmakevars.get_variables()) + + assert self.toolchain_file.is_absolute() + + def write(self) -> Path: + if not self.toolchain_file.parent.exists(): + self.toolchain_file.parent.mkdir(parents=True) + self.toolchain_file.write_text(self.generate()) + mlog.cmd_ci_include(self.toolchain_file.as_posix()) + return self.toolchain_file + + def get_cmake_args(self) -> T.List[str]: + args = ['-DCMAKE_TOOLCHAIN_FILE=' + self.toolchain_file.as_posix()] + if self.preload_file is not None: + args += ['-DMESON_PRELOAD_FILE=' + self.preload_file.as_posix()] + return args + + def generate(self) -> str: + res = dedent('''\ + ###################################### + ### AUTOMATICALLY GENERATED FILE ### + ###################################### + + # This file was generated from the configuration in the + # relevant meson machine file. See the meson documentation + # https://mesonbuild.com/Machine-files.html for more information + + if(DEFINED MESON_PRELOAD_FILE) + include("${MESON_PRELOAD_FILE}") + endif() + + ''') + + # Escape all \ in the values + for key, value in self.variables.items(): + self.variables[key] = [x.replace('\\', '/') for x in value] + + # Set variables from the current machine config + res += '# Variables from meson\n' + for key, value in self.variables.items(): + res += 'set(' + key + for i in value: + res += ' "{}"'.format(i) + + res += ')\n' + res += '\n' + + # Add the user provided toolchain file + user_file = self.properties.get_cmake_toolchain_file() + if user_file is not None: + res += dedent(''' + # Load the CMake toolchain file specified by the user + include("{}") + + '''.format(user_file.as_posix())) + + return res + + def get_defaults(self) -> T.Dict[str, T.List[str]]: + defaults = {} # type: T.Dict[str, T.List[str]] + + # Do nothing if the user does not want automatic defaults + if not self.properties.get_cmake_defaults(): + return defaults + + # Best effort to map the meson system name to CMAKE_SYSTEM_NAME, which + # is not trivial since CMake lacks a list of all supported + # CMAKE_SYSTEM_NAME values. + SYSTEM_MAP = { + 'android': 'Android', + 'linux': 'Linux', + 'windows': 'Windows', + 'freebsd': 'FreeBSD', + 'darwin': 'Darwin', + } # type: T.Dict[str, str] + + # Only set these in a cross build. Otherwise CMake will trip up in native + # builds and thing they are cross (which causes TRY_RUN() to break) + if self.env.is_cross_build(when_building_for=self.for_machine): + defaults['CMAKE_SYSTEM_NAME'] = [SYSTEM_MAP.get(self.minfo.system, self.minfo.system)] + defaults['CMAKE_SYSTEM_PROCESSOR'] = [self.minfo.cpu_family] + + defaults['CMAKE_SIZEOF_VOID_P'] = ['8' if self.minfo.is_64_bit else '4'] + + sys_root = self.properties.get_sys_root() + if sys_root: + defaults['CMAKE_SYSROOT'] = [sys_root] + + # Determine whether CMake the compiler test should be skipped + skip_check = self.properties.get_cmake_skip_compiler_test() == CMakeSkipCompilerTest.ALWAYS + if self.properties.get_cmake_skip_compiler_test() == CMakeSkipCompilerTest.DEP_ONLY and self.exec_scope == CMakeExecScope.DEPENDENCY: + skip_check = True + + def make_abs(exe: str) -> str: + if Path(exe).is_absolute(): + return exe + + p = shutil.which(exe) + if p is None: + return exe + return p + + # Set the compiler variables + for lang, comp_obj in self.compilers.items(): + exe_list = [make_abs(x) for x in comp_obj.get_exelist()] + comp_id = CMakeToolchain.meson_compiler_to_cmake_id(comp_obj) + comp_version = comp_obj.version.upper() + + prefix = 'CMAKE_{}_'.format(language_map.get(lang, lang.upper())) + + if not exe_list: + continue + elif len(exe_list) == 2: + defaults[prefix + 'COMPILER'] = [exe_list[1]] + defaults[prefix + 'COMPILER_LAUNCHER'] = [exe_list[0]] + else: + defaults[prefix + 'COMPILER'] = exe_list + if comp_obj.get_id() == 'clang-cl': + defaults['CMAKE_LINKER'] = comp_obj.get_linker_exelist() + + # Setting the variables after this check cause CMake to skip + # validating the compiler + if not skip_check: + continue + + defaults[prefix + 'COMPILER_ID'] = [comp_id] + defaults[prefix + 'COMPILER_VERSION'] = [comp_version] + #defaults[prefix + 'COMPILER_LOADED'] = ['1'] + defaults[prefix + 'COMPILER_FORCED'] = ['1'] + defaults[prefix + 'COMPILER_WORKS'] = ['TRUE'] + #defaults[prefix + 'ABI_COMPILED'] = ['TRUE'] + + return defaults + + @staticmethod + def meson_compiler_to_cmake_id(cobj: 'Compiler') -> str: + """Translate meson compiler's into CMAKE compiler ID's. + + Most of these can be handled by a simple table lookup, with a few + exceptions. + + Clang and Apple's Clang are both identified as "clang" by meson. To make + things more complicated gcc and vanilla clang both use Apple's ld64 on + macOS. The only way to know for sure is to do an isinstance() check. + """ + from ..compilers import (AppleClangCCompiler, AppleClangCPPCompiler, + AppleClangObjCCompiler, AppleClangObjCPPCompiler) + if isinstance(cobj, (AppleClangCCompiler, AppleClangCPPCompiler, + AppleClangObjCCompiler, AppleClangObjCPPCompiler)): + return 'AppleClang' + # If no mapping, try GNU and hope that the build files don't care + return _MESON_TO_CMAKE_MAPPING.get(cobj.get_id(), 'GNU') diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py index 95202fee2..2e56565a8 100644 --- a/mesonbuild/dependencies/base.py +++ b/mesonbuild/dependencies/base.py @@ -34,7 +34,7 @@ from .. import mesonlib from ..compilers import clib_langs from ..envconfig import get_env_var from ..environment import BinaryTable, Environment, MachineInfo -from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException +from ..cmake import CMakeExecutor, CMakeTraceParser, CMakeException, CMakeToolchain, CMakeExecScope from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args from ..mesonlib import Version, LibType @@ -1087,10 +1087,10 @@ class CMakeDependency(ExternalDependency): # AttributeError exceptions in derived classes self.traceparser = None # type: CMakeTraceParser - self.cmakebin = CMakeExecutor(environment, CMakeDependency.class_cmake_version, self.for_machine, silent=self.silent) + self.cmakebin = CMakeExecutor(environment, CMakeDependency.class_cmake_version, MachineChoice.BUILD, silent=self.silent) if not self.cmakebin.found(): self.cmakebin = None - msg = 'No CMake binary for machine %s not found. Giving up.' % self.for_machine + msg = 'No CMake binary for machine {} not found. Giving up.'.format(MachineChoice.BUILD) if self.required: raise DependencyException(msg) mlog.debug(msg) @@ -1136,12 +1136,14 @@ class CMakeDependency(ExternalDependency): gen_list += CMakeDependency.class_cmake_generators temp_parser = CMakeTraceParser(self.cmakebin.version(), self._get_build_dir()) + toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir()) + toolchain.write() for i in gen_list: mlog.debug('Try CMake generator: {}'.format(i if len(i) > 0 else 'auto')) # Prepare options - cmake_opts = temp_parser.trace_args() + ['.'] + cmake_opts = temp_parser.trace_args() + toolchain.get_cmake_args() + ['.'] cmake_opts += cm_args if len(i) > 0: cmake_opts = ['-G', i] + cmake_opts @@ -1320,6 +1322,8 @@ class CMakeDependency(ExternalDependency): # Map the components comp_mapped = self._map_component_list(modules, components) + toolchain = CMakeToolchain(self.env, self.for_machine, CMakeExecScope.DEPENDENCY, self._get_build_dir()) + toolchain.write() for i in gen_list: mlog.debug('Try CMake generator: {}'.format(i if len(i) > 0 else 'auto')) @@ -1331,6 +1335,7 @@ class CMakeDependency(ExternalDependency): cmake_opts += ['-DCOMPS={}'.format(';'.join([x[0] for x in comp_mapped]))] cmake_opts += args cmake_opts += self.traceparser.trace_args() + cmake_opts += toolchain.get_cmake_args() cmake_opts += self._extra_cmake_opts() cmake_opts += ['.'] if len(i) > 0: @@ -1537,6 +1542,13 @@ class CMakeDependency(ExternalDependency): # Setup the CMake build environment and return the "build" directory build_dir = self._get_build_dir() + # Remove old CMake cache so we can try out multiple generators + cmake_cache = build_dir / 'CMakeCache.txt' + cmake_files = build_dir / 'CMakeFiles' + if cmake_cache.exists(): + cmake_cache.unlink() + shutil.rmtree(cmake_files.as_posix(), ignore_errors=True) + # Insert language parameters into the CMakeLists.txt and write new CMakeLists.txt cmake_txt = mesondata['dependencies/data/' + cmake_file].data @@ -1552,10 +1564,10 @@ class CMakeDependency(ExternalDependency): if not cmake_language: cmake_language += ['NONE'] - cmake_txt = """ -cmake_minimum_required(VERSION ${{CMAKE_VERSION}}) -project(MesonTemp LANGUAGES {}) -""".format(' '.join(cmake_language)) + cmake_txt + cmake_txt = textwrap.dedent(""" + cmake_minimum_required(VERSION ${{CMAKE_VERSION}}) + project(MesonTemp LANGUAGES {}) + """).format(' '.join(cmake_language)) + cmake_txt cm_file = build_dir / 'CMakeLists.txt' cm_file.write_text(cmake_txt) @@ -1565,7 +1577,7 @@ project(MesonTemp LANGUAGES {}) def _call_cmake(self, args, cmake_file: str, env=None): build_dir = self._setup_cmake_dir(cmake_file) - return self.cmakebin.call_with_fake_build(args, build_dir, env=env) + return self.cmakebin.call(args, build_dir, env=env) @staticmethod def get_methods(): diff --git a/mesonbuild/envconfig.py b/mesonbuild/envconfig.py index 8eaf9e4f4..13d0ba507 100644 --- a/mesonbuild/envconfig.py +++ b/mesonbuild/envconfig.py @@ -14,10 +14,12 @@ import os, subprocess import typing as T +from enum import Enum from . import mesonlib from .mesonlib import EnvironmentException, MachineChoice, PerMachine, split_args from . import mlog +from ._pathlib import Path _T = T.TypeVar('_T') @@ -83,6 +85,12 @@ CPU_FAMILES_64_BIT = [ 'x86_64', ] +class CMakeSkipCompilerTest(Enum): + ALWAYS = 'always' + NEVER = 'never' + DEP_ONLY = 'dep_only' + + def get_env_var_pair(for_machine: MachineChoice, is_cross: bool, var_name: str) -> T.Optional[T.Tuple[str, str]]: @@ -120,9 +128,9 @@ def get_env_var(for_machine: MachineChoice, class Properties: def __init__( self, - properties: T.Optional[T.Dict[str, T.Union[str, T.List[str]]]] = None, + properties: T.Optional[T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = None, ): - self.properties = properties or {} # type: T.Dict[str, T.Union[str, T.List[str]]] + self.properties = properties or {} # type: T.Dict[str, T.Union[str, bool, int, T.List[str]]] def has_stdlib(self, language: str) -> bool: return language + '_stdlib' in self.properties @@ -131,19 +139,68 @@ class Properties: # true, but without heterogenious dict annotations it's not practical to # narrow them def get_stdlib(self, language: str) -> T.Union[str, T.List[str]]: - return self.properties[language + '_stdlib'] - - def get_root(self) -> T.Optional[T.Union[str, T.List[str]]]: - return self.properties.get('root', None) - - def get_sys_root(self) -> T.Optional[T.Union[str, T.List[str]]]: - return self.properties.get('sys_root', None) + stdlib = self.properties[language + '_stdlib'] + if isinstance(stdlib, str): + return stdlib + assert isinstance(stdlib, list) + for i in stdlib: + assert isinstance(i, str) + return stdlib + + def get_root(self) -> T.Optional[str]: + root = self.properties.get('root', None) + assert root is None or isinstance(root, str) + return root + + def get_sys_root(self) -> T.Optional[str]: + sys_root = self.properties.get('sys_root', None) + assert sys_root is None or isinstance(sys_root, str) + return sys_root def get_pkg_config_libdir(self) -> T.Optional[T.List[str]]: p = self.properties.get('pkg_config_libdir', None) if p is None: return p - return mesonlib.listify(p) + res = mesonlib.listify(p) + for i in res: + assert isinstance(i, str) + return res + + def get_cmake_defaults(self) -> bool: + if 'cmake_defaults' not in self.properties: + return True + res = self.properties['cmake_defaults'] + assert isinstance(res, bool) + return res + + def get_cmake_toolchain_file(self) -> T.Optional[Path]: + if 'cmake_toolchain_file' not in self.properties: + return None + raw = self.properties['cmake_toolchain_file'] + assert isinstance(raw, str) + cmake_toolchain_file = Path(raw) + if not cmake_toolchain_file.is_absolute(): + raise EnvironmentException('cmake_toolchain_file ({}) is not absolute'.format(raw)) + return cmake_toolchain_file + + def get_cmake_skip_compiler_test(self) -> CMakeSkipCompilerTest: + if 'cmake_skip_compiler_test' not in self.properties: + return CMakeSkipCompilerTest.DEP_ONLY + raw = self.properties['cmake_skip_compiler_test'] + assert isinstance(raw, str) + try: + return CMakeSkipCompilerTest(raw) + except ValueError: + raise EnvironmentException( + '"{}" is not a valid value for cmake_skip_compiler_test. Supported values are {}' + .format(raw, [e.value for e in CMakeSkipCompilerTest])) + + def get_cmake_use_exe_wrapper(self) -> bool: + if 'cmake_use_exe_wrapper' not in self.properties: + return True + res = self.properties['cmake_use_exe_wrapper'] + assert isinstance(res, bool) + return res def __eq__(self, other: object) -> bool: if isinstance(other, type(self)): @@ -151,15 +208,15 @@ class Properties: return NotImplemented # TODO consider removing so Properties is less freeform - def __getitem__(self, key: str) -> T.Any: + def __getitem__(self, key: str) -> T.Union[str, bool, int, T.List[str]]: return self.properties[key] # TODO consider removing so Properties is less freeform - def __contains__(self, item: T.Any) -> bool: + def __contains__(self, item: T.Union[str, bool, int, T.List[str]]) -> bool: return item in self.properties # TODO consider removing, for same reasons as above - def get(self, key: str, default: T.Any = None) -> T.Any: + def get(self, key: str, default: T.Union[str, bool, int, T.List[str]] = None) -> T.Union[str, bool, int, T.List[str]]: return self.properties.get(key, default) class MachineInfo: @@ -406,3 +463,17 @@ class BinaryTable: if command is not None and (len(command) == 0 or len(command[0].strip()) == 0): command = None return command + +class CMakeVariables: + def __init__(self, variables: T.Optional[T.Dict[str, T.Any]] = None) -> None: + variables = variables or {} + self.variables = {} # type: T.Dict[str, T.List[str]] + + for key, value in variables.items(): + value = mesonlib.listify(value) + for i in value: + assert isinstance(i, str) + self.variables[key] = value + + def get_variables(self) -> T.Dict[str, T.List[str]]: + return self.variables diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 33cc3a5ba..394ef6a61 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -30,6 +30,7 @@ from . import mlog from .envconfig import ( BinaryTable, MachineInfo, Properties, known_cpu_families, get_env_var_pair, + CMakeVariables, ) from . import compilers from .compilers import ( @@ -569,14 +570,17 @@ class Environment: # Stores machine infos, the only *three* machine one because we have a # target machine info on for the user (Meson never cares about the # target machine.) - machines = PerThreeMachineDefaultable() + machines = PerThreeMachineDefaultable() # type: PerMachineDefaultable[MachineInfo] # Similar to coredata.compilers, but lower level in that there is no # meta data, only names/paths. - binaries = PerMachineDefaultable() + binaries = PerMachineDefaultable() # type: PerMachineDefaultable[BinaryTable] # Misc other properties about each machine. - properties = PerMachineDefaultable() + properties = PerMachineDefaultable() # type: PerMachineDefaultable[Properties] + + # CMake toolchain variables + cmakevars = PerMachineDefaultable() # type: PerMachineDefaultable[CMakeVariables] # We only need one of these as project options are not per machine user_options = collections.defaultdict(dict) # type: T.DefaultDict[str, T.Dict[str, object]] @@ -653,6 +657,7 @@ class Environment: config = coredata.parse_machine_files(self.coredata.config_files) binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) + cmakevars.build = CMakeVariables(config.get('cmake', {})) # Don't run this if there are any cross files, we don't want to use # the native values if we're doing a cross build @@ -674,6 +679,7 @@ class Environment: config = coredata.parse_machine_files(self.coredata.cross_files) properties.host = Properties(config.get('properties', {})) binaries.host = BinaryTable(config.get('binaries', {})) + cmakevars.host = CMakeVariables(config.get('cmake', {})) if 'host_machine' in config: machines.host = MachineInfo.from_literal(config['host_machine']) if 'target_machine' in config: @@ -694,6 +700,7 @@ class Environment: self.machines = machines.default_missing() self.binaries = binaries.default_missing() self.properties = properties.default_missing() + self.cmakevars = cmakevars.default_missing() self.user_options = user_options self.meson_options = meson_options.default_missing() self.base_options = _base_options diff --git a/run_unittests.py b/run_unittests.py index 160173080..22e7cdc48 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -1239,6 +1239,7 @@ class InternalTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmpdir: with chdir(tmpdir): env = get_fake_env() + env.scratch_dir = tmpdir f = b.DependencyFactory( 'test_dep', From b27af7e4654c3b2fe8c68a560c99fbffbd22789b Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 5 Oct 2020 20:45:21 +0200 Subject: [PATCH 2/3] cmake: Add cross tests --- .github/workflows/nonative.yml | 2 +- .travis.yml | 4 ++-- ci/travis_script.sh | 5 +++++ cross/linux-mingw-w64-32bit.json | 7 +++++++ cross/linux-mingw-w64-32bit.txt | 8 ++++++++ cross/linux-mingw-w64-64bit.json | 7 +++++++ cross/linux-mingw-w64-64bit.txt | 8 ++++++++ cross/ubuntu-armhf.json | 5 +++++ run_cross_test.py | 20 +++++++++++++++---- run_project_tests.py | 1 + .../subprojects/cmMod/CMakeLists.txt | 2 +- test cases/cmake/2 advanced/test.json | 2 -- .../cmake_project/CMakeLists.txt | 0 .../meson.build | 0 .../projectConfig.cmake.in | 0 .../test.json | 0 .../23 cmake toolchain/CMakeToolchain.cmake | 1 + .../cmake/23 cmake toolchain/meson.build | 9 +++++++++ .../23 cmake toolchain/nativefile.ini.in | 8 ++++++++ .../subprojects/cmMod/CMakeLists.txt | 11 ++++++++++ .../subprojects/cmMod/CMakeLists.txt | 2 +- test cases/cmake/3 advanced no dep/test.json | 3 --- .../subprojects/cmCodeGen/CMakeLists.txt | 1 + test cases/cmake/7 cmake options/test.json | 3 +++ 24 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 cross/linux-mingw-w64-32bit.json create mode 100644 cross/linux-mingw-w64-64bit.json create mode 100644 cross/ubuntu-armhf.json rename test cases/cmake/{211 cmake module => 22 cmake module}/cmake_project/CMakeLists.txt (100%) rename test cases/cmake/{211 cmake module => 22 cmake module}/meson.build (100%) rename test cases/cmake/{211 cmake module => 22 cmake module}/projectConfig.cmake.in (100%) rename test cases/cmake/{211 cmake module => 22 cmake module}/test.json (100%) create mode 100644 test cases/cmake/23 cmake toolchain/CMakeToolchain.cmake create mode 100644 test cases/cmake/23 cmake toolchain/meson.build create mode 100644 test cases/cmake/23 cmake toolchain/nativefile.ini.in create mode 100644 test cases/cmake/23 cmake toolchain/subprojects/cmMod/CMakeLists.txt diff --git a/.github/workflows/nonative.yml b/.github/workflows/nonative.yml index 59386c530..44eeb9e52 100644 --- a/.github/workflows/nonative.yml +++ b/.github/workflows/nonative.yml @@ -16,4 +16,4 @@ jobs: apt-get -y autoremove - uses: actions/checkout@v2 - name: Run tests - run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS --cross ubuntu-armhf.txt --cross-only' + run: bash -c 'source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py $CI_ARGS --cross ubuntu-armhf.json --cross-only' diff --git a/.travis.yml b/.travis.yml index 22d76e7ea..ab317be15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,10 +34,10 @@ matrix: # Also hijack one cross build to test long commandline handling codepath (and avoid overloading Travis) - os: linux compiler: gcc - env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt" MESON_RSP_THRESHOLD=0 + env: RUN_TESTS_ARGS="--cross ubuntu-armhf.json --cross linux-mingw-w64-64bit.json" MESON_RSP_THRESHOLD=0 - os: linux compiler: gcc - env: RUN_TESTS_ARGS="--cross ubuntu-armhf.txt --cross linux-mingw-w64-64bit.txt" MESON_ARGS="--unity=on" + env: RUN_TESTS_ARGS="--cross ubuntu-armhf.json --cross linux-mingw-w64-64bit.json" MESON_ARGS="--unity=on" before_install: - python ./skip_ci.py --base-branch-env=TRAVIS_BRANCH --is-pull-env=TRAVIS_PULL_REQUEST diff --git a/ci/travis_script.sh b/ci/travis_script.sh index bdfd4c202..7e26b5232 100755 --- a/ci/travis_script.sh +++ b/ci/travis_script.sh @@ -31,6 +31,11 @@ fi source /ci/env_vars.sh cd /root +update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix +update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix +update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix +update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix + ./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS #./upload.sh diff --git a/cross/linux-mingw-w64-32bit.json b/cross/linux-mingw-w64-32bit.json new file mode 100644 index 000000000..476111183 --- /dev/null +++ b/cross/linux-mingw-w64-32bit.json @@ -0,0 +1,7 @@ +{ + "file": "linux-mingw-w64-32bit.txt", + "tests": ["common", "cmake"], + "env": { + "WINEPATH": "/usr/lib/gcc/i686-w64-mingw32/9.2-posix;/usr/i686-w64-mingw32/bin;/usr/i686-w64-mingw32/lib" + } +} diff --git a/cross/linux-mingw-w64-32bit.txt b/cross/linux-mingw-w64-32bit.txt index c2ea605b8..a62f57f9b 100644 --- a/cross/linux-mingw-w64-32bit.txt +++ b/cross/linux-mingw-w64-32bit.txt @@ -19,3 +19,11 @@ system = 'windows' cpu_family = 'x86' cpu = 'i686' endian = 'little' + +[cmake] + +CMAKE_BUILD_WITH_INSTALL_RPATH = 'ON' +CMAKE_FIND_ROOT_PATH_MODE_PROGRAM = 'NEVER' +CMAKE_FIND_ROOT_PATH_MODE_LIBRARY = 'ONLY' +CMAKE_FIND_ROOT_PATH_MODE_INCLUDE = 'ONLY' +CMAKE_FIND_ROOT_PATH_MODE_PACKAGE = 'ONLY' diff --git a/cross/linux-mingw-w64-64bit.json b/cross/linux-mingw-w64-64bit.json new file mode 100644 index 000000000..df344da9d --- /dev/null +++ b/cross/linux-mingw-w64-64bit.json @@ -0,0 +1,7 @@ +{ + "file": "linux-mingw-w64-64bit.txt", + "tests": ["common", "cmake"], + "env": { + "WINEPATH": "/usr/lib/gcc/x86_64-w64-mingw32/9.2-posix;/usr/x86_64-w64-mingw32/bin;/usr/x86_64-w64-mingw32/lib" + } +} diff --git a/cross/linux-mingw-w64-64bit.txt b/cross/linux-mingw-w64-64bit.txt index 1c5c00238..36d73500a 100644 --- a/cross/linux-mingw-w64-64bit.txt +++ b/cross/linux-mingw-w64-64bit.txt @@ -18,3 +18,11 @@ system = 'windows' cpu_family = 'x86_64' cpu = 'x86_64' endian = 'little' + +[cmake] + +CMAKE_BUILD_WITH_INSTALL_RPATH = 'ON' +CMAKE_FIND_ROOT_PATH_MODE_PROGRAM = 'NEVER' +CMAKE_FIND_ROOT_PATH_MODE_LIBRARY = 'ONLY' +CMAKE_FIND_ROOT_PATH_MODE_INCLUDE = 'ONLY' +CMAKE_FIND_ROOT_PATH_MODE_PACKAGE = 'ONLY' diff --git a/cross/ubuntu-armhf.json b/cross/ubuntu-armhf.json new file mode 100644 index 000000000..40f5619c2 --- /dev/null +++ b/cross/ubuntu-armhf.json @@ -0,0 +1,5 @@ +{ + "file": "ubuntu-armhf.txt", + "tests": ["common"], + "env": {} +} diff --git a/run_cross_test.py b/run_cross_test.py index 836cf3165..5ce3e5528 100755 --- a/run_cross_test.py +++ b/run_cross_test.py @@ -23,10 +23,13 @@ import argparse import subprocess from mesonbuild import mesonlib from mesonbuild.coredata import version as meson_version +from pathlib import Path +import json +import os -def runtests(cross_file, failfast, cross_only): - tests = ['--only', 'common'] +def runtests(cross_file, failfast, cross_only, test_list, env=None): + tests = ['--only'] + test_list if not cross_only: tests.append('native') cmd = mesonlib.python_command + ['run_project_tests.py', '--backend', 'ninja'] @@ -36,7 +39,7 @@ def runtests(cross_file, failfast, cross_only): cmd += ['--cross-file', cross_file] if cross_only: cmd += ['--native-file', 'cross/none.txt'] - return subprocess.call(cmd) + return subprocess.call(cmd, env=env) def main(): parser = argparse.ArgumentParser() @@ -44,7 +47,16 @@ def main(): parser.add_argument('--cross-only', action='store_true') parser.add_argument('cross_file') options = parser.parse_args() - return runtests(options.cross_file, options.failfast, options.cross_only) + cf_path = Path(options.cross_file) + try: + data = json.loads(cf_path.read_text()) + real_cf = cf_path.resolve().parent / data['file'] + assert real_cf.exists() + env = os.environ.copy() + env.update(data['env']) + return runtests(real_cf.as_posix(), options.failfast, options.cross_only, data['tests'], env=env) + except Exception: + return runtests(options.cross_file, options.failfast, options.cross_only, ['common']) if __name__ == '__main__': print('Meson build system', meson_version, 'Cross Tests') diff --git a/run_project_tests.py b/run_project_tests.py index 87499570e..037ba4218 100755 --- a/run_project_tests.py +++ b/run_project_tests.py @@ -1163,6 +1163,7 @@ def check_format(): '.dub', # external deps are here '.pytest_cache', 'meson-logs', 'meson-private', + 'work area', '.eggs', '_cache', # e.g. .mypy_cache 'venv', # virtualenvs have DOS line endings } diff --git a/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt b/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt index c9b2a208f..7fce89e25 100644 --- a/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt +++ b/test cases/cmake/2 advanced/subprojects/cmMod/CMakeLists.txt @@ -25,4 +25,4 @@ target_link_libraries(testEXE cmModLib) target_compile_definitions(cmModLibStatic PUBLIC CMMODLIB_STATIC_DEFINE) -install(TARGETS cmModLib testEXE LIBRARY DESTINATION lib RUNTIME DESTINATION bin) +install(TARGETS testEXE LIBRARY DESTINATION lib RUNTIME DESTINATION bin) diff --git a/test cases/cmake/2 advanced/test.json b/test cases/cmake/2 advanced/test.json index ff3d5a73d..e2d9c051f 100644 --- a/test cases/cmake/2 advanced/test.json +++ b/test cases/cmake/2 advanced/test.json @@ -1,7 +1,5 @@ { "installed": [ - {"type": "expr", "file": "usr/?lib/libcm_cmModLib?so"}, - {"type": "implib", "file": "usr/lib/libcm_cmModLib"}, {"type": "exe", "file": "usr/bin/cm_testEXE"} ], "tools": { diff --git a/test cases/cmake/211 cmake module/cmake_project/CMakeLists.txt b/test cases/cmake/22 cmake module/cmake_project/CMakeLists.txt similarity index 100% rename from test cases/cmake/211 cmake module/cmake_project/CMakeLists.txt rename to test cases/cmake/22 cmake module/cmake_project/CMakeLists.txt diff --git a/test cases/cmake/211 cmake module/meson.build b/test cases/cmake/22 cmake module/meson.build similarity index 100% rename from test cases/cmake/211 cmake module/meson.build rename to test cases/cmake/22 cmake module/meson.build diff --git a/test cases/cmake/211 cmake module/projectConfig.cmake.in b/test cases/cmake/22 cmake module/projectConfig.cmake.in similarity index 100% rename from test cases/cmake/211 cmake module/projectConfig.cmake.in rename to test cases/cmake/22 cmake module/projectConfig.cmake.in diff --git a/test cases/cmake/211 cmake module/test.json b/test cases/cmake/22 cmake module/test.json similarity index 100% rename from test cases/cmake/211 cmake module/test.json rename to test cases/cmake/22 cmake module/test.json diff --git a/test cases/cmake/23 cmake toolchain/CMakeToolchain.cmake b/test cases/cmake/23 cmake toolchain/CMakeToolchain.cmake new file mode 100644 index 000000000..ab5fbace1 --- /dev/null +++ b/test cases/cmake/23 cmake toolchain/CMakeToolchain.cmake @@ -0,0 +1 @@ +set(MESON_TEST_VAR2 VAR2) diff --git a/test cases/cmake/23 cmake toolchain/meson.build b/test cases/cmake/23 cmake toolchain/meson.build new file mode 100644 index 000000000..98f8d21df --- /dev/null +++ b/test cases/cmake/23 cmake toolchain/meson.build @@ -0,0 +1,9 @@ +project('cmake toolchain test', ['c', 'cpp']) + +if meson.is_cross_build() + error('MESON_SKIP_TEST: skip this on cross builds') +endif + +cm = import('cmake') + +sub_pro = cm.subproject('cmMod') diff --git a/test cases/cmake/23 cmake toolchain/nativefile.ini.in b/test cases/cmake/23 cmake toolchain/nativefile.ini.in new file mode 100644 index 000000000..2cd6e947a --- /dev/null +++ b/test cases/cmake/23 cmake toolchain/nativefile.ini.in @@ -0,0 +1,8 @@ +[properties] + +cmake_toolchain_file = '@MESON_TEST_ROOT@/CMakeToolchain.cmake' + +[cmake] + +MESON_TEST_VAR1 = 'VAR1 space' +MESON_TEST_VAR2 = 'VAR2 error' diff --git a/test cases/cmake/23 cmake toolchain/subprojects/cmMod/CMakeLists.txt b/test cases/cmake/23 cmake toolchain/subprojects/cmMod/CMakeLists.txt new file mode 100644 index 000000000..8aeabc2a2 --- /dev/null +++ b/test cases/cmake/23 cmake toolchain/subprojects/cmMod/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.5) + +project(cmMod) + +if(NOT "${MESON_TEST_VAR1}" STREQUAL "VAR1 space") + message(FATAL_ERROR "MESON_TEST_VAR1 -- '${MESON_TEST_VAR1}' != 'VAR1 space'") +endif() + +if(NOT "${MESON_TEST_VAR2}" STREQUAL "VAR2") + message(FATAL_ERROR "MESON_TEST_VAR2 -- '${MESON_TEST_VAR2}' != 'VAR2'") +endif() diff --git a/test cases/cmake/3 advanced no dep/subprojects/cmMod/CMakeLists.txt b/test cases/cmake/3 advanced no dep/subprojects/cmMod/CMakeLists.txt index 4c782cb38..026d4c103 100644 --- a/test cases/cmake/3 advanced no dep/subprojects/cmMod/CMakeLists.txt +++ b/test cases/cmake/3 advanced no dep/subprojects/cmMod/CMakeLists.txt @@ -23,4 +23,4 @@ target_link_libraries(testEXE2 cmModLib) target_compile_definitions(cmModLibStatic PUBLIC CMMODLIB_STATIC_DEFINE) -install(TARGETS cmModLib testEXE testEXE2 LIBRARY DESTINATION lib RUNTIME DESTINATION bin) +install(TARGETS testEXE testEXE2 LIBRARY DESTINATION lib RUNTIME DESTINATION bin) diff --git a/test cases/cmake/3 advanced no dep/test.json b/test cases/cmake/3 advanced no dep/test.json index af25a8ec6..4b266c320 100644 --- a/test cases/cmake/3 advanced no dep/test.json +++ b/test cases/cmake/3 advanced no dep/test.json @@ -1,8 +1,5 @@ { "installed": [ - {"type": "expr", "file": "usr/?lib/libcm_cmModLib?so"}, - {"type": "implib", "file": "usr/lib/libcm_cmModLib"}, - {"type": "pdb", "file": "usr/bin/cm_cmModLib"}, {"type": "pdb", "file": "usr/bin/cm_testEXE"}, {"type": "exe", "file": "usr/bin/cm_testEXE"}, {"type": "pdb", "file": "usr/bin/cm_testEXE2"}, diff --git a/test cases/cmake/4 code gen/subprojects/cmCodeGen/CMakeLists.txt b/test cases/cmake/4 code gen/subprojects/cmCodeGen/CMakeLists.txt index 268743c73..ff50e54d5 100644 --- a/test cases/cmake/4 code gen/subprojects/cmCodeGen/CMakeLists.txt +++ b/test cases/cmake/4 code gen/subprojects/cmCodeGen/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.7) +project(CMCodeGen) set(CMAKE_CXX_STANDARD 14) add_executable(genA main.cpp) diff --git a/test cases/cmake/7 cmake options/test.json b/test cases/cmake/7 cmake options/test.json index 046e2ee4c..f9f0b05a7 100644 --- a/test cases/cmake/7 cmake options/test.json +++ b/test cases/cmake/7 cmake options/test.json @@ -3,6 +3,9 @@ "options": { "cmake_prefix_path": [ { "val": ["val1", "val2"] } + ], + "build.cmake_prefix_path": [ + { "val": ["val1", "val2"] } ] } } From f5c9bf96b370832fc1a6e50771e2c171de0cd79d Mon Sep 17 00:00:00 2001 From: Daniel Mensinger Date: Mon, 5 Oct 2020 20:45:38 +0200 Subject: [PATCH 3/3] cmake: Add cross docs --- docs/markdown/CMake-module.md | 37 +++++++++++++++++ docs/markdown/Machine-files.md | 57 +++++++++++++++++++++++++++ docs/markdown/snippets/cmake_cross.md | 8 ++++ 3 files changed, 102 insertions(+) create mode 100644 docs/markdown/snippets/cmake_cross.md diff --git a/docs/markdown/CMake-module.md b/docs/markdown/CMake-module.md index fc6157ed7..48c3d75b1 100644 --- a/docs/markdown/CMake-module.md +++ b/docs/markdown/CMake-module.md @@ -176,6 +176,43 @@ Options that are not set won't affect the generated subproject. So, if for instance, `set_install` was not called then the values extracted from CMake will be used. +### Cross compilation + +*New in 0.56.0* + +Meson will try to automatically guess most of the required CMake toolchain +variables from existing entries in the cross and native files. These variables +will be stored in an automatically generate CMake toolchain file in the build +directory. The remaining variables that can't be guessed can be added by the +user in the `[cmake]` cross/native file section (*new in 0.56.0*). + +Adding a manual CMake toolchain file is also supported with the +`cmake_toolchain_file` setting in the `[properties]` section. Directly setting +a CMake toolchain file with `-DCMAKE_TOOLCHAIN_FILE=/path/to/some/Toolchain.cmake` +in the `meson.build` is **not** supported since the automatically generated +toolchain file is also used by Meson to inject arbitrary code into CMake to +enable the CMake subproject support. + +The closest configuration to only using a manual CMake toolchain file would be +to set these options in the machine file: + +```ini +[properties] + +cmake_toolchain_file = '/path/to/some/Toolchain.cmake' +cmake_defaults = false + +[cmake] + +# No entries in this section +``` + +This will result in a toolchain file with just the bare minimum to enable the +CMake subproject support and `include()` the `cmake_toolchain_file` as the +last instruction. + +For more information see the [cross and native file specification](Machine-files.md). + ## CMake configuration files ### cmake.write_basic_package_version_file() diff --git a/docs/markdown/Machine-files.md b/docs/markdown/Machine-files.md index ab450cc49..72d2e0cfc 100644 --- a/docs/markdown/Machine-files.md +++ b/docs/markdown/Machine-files.md @@ -46,6 +46,7 @@ The following sections are allowed: - binaries - paths - properties +- cmake - project options - built-in options @@ -203,6 +204,62 @@ section may contain random key value pairs accessed using the properties section has been deprecated, and should be put in the built-in options section. +#### Supported properties + +This is a non exhaustive list of supported variables in the `[properties]` +section. + +- `cmake_toolchain_file` specifies an absoulte path to an already existing + CMake toolchain file that will be loaded with `include()` as the last + instruction of the automatically generated CMake toolchain file from meson. + (*new in 0.56.0*) +- `cmake_defaults` is a boolean that specifies whether meson should automatically + generate default toolchain varaibles from other sections (`binaries`, + `host_machine`, etc.) in the machine file. Defaults are always overwritten + by variables set in the `[cmake]` section. The default is `true`. (*new in 0.56.0*) +- `cmake_skip_compiler_test` is an enum that specifies when meson should + automatically generate toolchain variables to skip the CMake compiler + sanity checks. This only has an effect if `cmake_defaults` is `true`. + Supported values are `always`, `never`, `dep_only`. The default is `dep_only`. + (*new in 0.56.0*) +- `cmake_use_exe_wrapper` is a boolean that controlls whether to use the + `exe_wrapper` specified in `[binaries]` to run generated executables in CMake + subprojects. This setting has no effect if the `exe_wrapper` was not specified. + The default value is `true`. (*new in 0.56.0*) + +### CMake variables + +*New in 0.56.0* + +All variables set in the `[cmake]` section will be added to the generate CMake +toolchain file used for both CMake dependencies and CMake subprojects. The type +of each entry must be either a string or a list of strings. + +**Note:** All occurances of `\` in the value of all keys will be replaced with + a `/` since CMake has a lot of issues with correctly escaping `\` when + dealing with variables (even in cases where a path in `CMAKE_C_COMPILER` + is correctly escaped, CMake will still trip up internaly for instance) + + A custom toolchain file should be used (via the `cmake_toolchain_file` + property) if `\` support is required. + +```ini +[cmake] + +CMAKE_C_COMPILER = '/usr/bin/gcc' +CMAKE_CXX_COMPILER = 'C:\\user\\bin\\g++' +CMAKE_SOME_VARIABLE = ['some', 'value with spaces'] +``` + +For instance, the `[cmake]` section from above will generate the following +code in the CMake toolchain file: + +```cmake +set(CMAKE_C_COMPILER "/usr/bin/gcc") +set(CMAKE_C_COMPILER "C:/usr/bin/g++") +set(CMAKE_SOME_VARIABLE "some" "value with spaces") +``` + ### Project specific options *New in 0.56.0* diff --git a/docs/markdown/snippets/cmake_cross.md b/docs/markdown/snippets/cmake_cross.md new file mode 100644 index 000000000..249c95f6a --- /dev/null +++ b/docs/markdown/snippets/cmake_cross.md @@ -0,0 +1,8 @@ +## CMake subproject cross compilation support + +Meson now supports cross compilation for CMake subprojects. Meson will try to +automatically guess most of the required CMake toolchain variables from existing +entries in the cross and native files. These variables will be stored in an +automatically generate CMake toolchain file in the build directory. The +remaining variables that can't be guessed can be added by the user in the +new `[cmake]` cross/native file section.