From e257a870fe5e676c55a2282b0e7fc9be34bba2ac Mon Sep 17 00:00:00 2001 From: Jussi Pakkanen Date: Sat, 19 Feb 2022 18:17:28 +0200 Subject: [PATCH] Add new env2cross command. --- docs/markdown/snippets/env2cross.md | 40 +++ mesonbuild/mesonmain.py | 4 +- mesonbuild/scripts/env2mfile.py | 368 ++++++++++++++++++++++++++++ 3 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 docs/markdown/snippets/env2cross.md create mode 100755 mesonbuild/scripts/env2mfile.py diff --git a/docs/markdown/snippets/env2cross.md b/docs/markdown/snippets/env2cross.md new file mode 100644 index 000000000..bb53145ce --- /dev/null +++ b/docs/markdown/snippets/env2cross.md @@ -0,0 +1,40 @@ +## Experimental command to convert environments to cross files + +Meson has a new command `env2mfile` that can be used to convert +"environment variable based" cross and native compilation environments +to Meson machine files. This is especially convenient for e.g. distro +packagers so they can easily generate unambiguous configuration files +for packge building. + +As an example here's how you would generate a cross file that takes +its settings from the `CC`, `CXX`, `CFLAGS` etc environment variables. + + meson env2mfile --cross --system=baremetal --cpu=armv7 --cpu-family=arm -o armcross.txt + +The command also has support for generating Debian build files using +system introspection: + + meson env2mfile --cross --debarch armhf -o debarmhf_cross.txt + +Note how you don't need to specify any system details, the command +gets them transparently via `dpkg-architecture`. + +Creating a native file is done in the same way: + + meson env2mfile --native -o current_system.txt + +This system will detect if the `_FOR_BUILD` environment variables are +enabled and then uses them as needed. + +With this you should be able to convert any envvar-based cross build +setup to cross and native files and then use those. Thit means, among +other things, that you can then run your compilations from any shell, +not just the special one that has all the environment variables set. + +As this functionality is still a bit in flux, the specific behaviour +and command line arguments to use are subject to change. Because of +this the main documentation has not yet been updated. + +Please try this for your use cases and report to us if it is working. +Patches to make the autodetection work on other distros and platforms +are also welcome. diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index 93cb8b0b7..89816ec4d 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -30,7 +30,7 @@ from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, ms from .mesonlib import MesonException, MesonBugException from .environment import detect_msys2_arch from .wrap import wraptool - +from .scripts import env2mfile # Note: when adding arguments, please also add them to the completion # scripts in $MESONSRC/data/shell-completions/ @@ -70,6 +70,8 @@ class CommandLineParser: help_msg='Build the project') self.add_command('devenv', mdevenv.add_arguments, mdevenv.run, help_msg='Run commands in developer environment') + self.add_command('env2mfile', env2mfile.add_arguments, env2mfile.run, + help_msg='Convert current environment to a cross or native file') # Hidden commands self.add_command('runpython', self.add_runpython_arguments, self.run_runpython_command, diff --git a/mesonbuild/scripts/env2mfile.py b/mesonbuild/scripts/env2mfile.py new file mode 100755 index 000000000..9441402e8 --- /dev/null +++ b/mesonbuild/scripts/env2mfile.py @@ -0,0 +1,368 @@ +# Copyright 2022 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, os, subprocess, shutil +import shlex + +import argparse +from typing import TextIO, Dict, List, Union, Tuple, Any, Optional + +from .. import mlog + +UNIXY_ENVVARS_COMPILER = {'c': 'CC', + 'cpp': 'CXX', + 'objc': 'OBJCC', + 'objcpp': 'OBJCXX', + 'fortran': 'FC', + 'rust': 'RUSTC', + 'vala': 'VALAC', + 'cs': 'CSC', + } + +UNIXY_ENVVARS_TOOLS = {'ar': 'AR', + 'strip': 'STRIP', + 'windres': 'WINDRES', + 'pkgconfig': 'PKG_CONFIG', + 'vapigen': 'VAPIGEN', + 'cmake': 'CMAKE', + 'qmake': 'QMAKE', + } + +UNIXY_ENVVARS_FLAGS = {'c': 'CFLAGS', + 'cpp': 'CXXFLAGS', + 'objc': 'OBJCFLAGS', + 'objcpp': 'OBJCXXFLAGS', + 'fortran': 'FFLAGS', + 'rust': 'RUSTFLAGS', + 'vala': 'VALAFLAGS', + 'cs': 'CSFLAGS', # This one might not be standard. + } + +TYPICAL_UNIXY_COMPILER_NAMES = {'c': ['cc', 'gcc', 'clang'], + 'cpp': ['c++', 'g++', 'clang++'], + 'objc': ['objc', 'clang'], + 'objcpp': ['objcpp', 'clang++'], + 'fortran': ['gfortran'], + } + +LANGS_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcxx'} + +def has_for_build() -> bool: + for cenv in UNIXY_ENVVARS_COMPILER.values(): + if os.environ.get(cenv + '_FOR_BUILD'): + return True + return False + +def add_arguments(parser: 'argparse.ArgumentParser') -> None: + parser.add_argument('--debarch', default=None, + help='The dpkg architecture to generate.') + parser.add_argument('--gccsuffix', default="", + help='A particular gcc version suffix if necessary.') + parser.add_argument('-o', required=True, dest='outfile', + help='The output file.') + parser.add_argument('--cross', default=False, action='store_true', + help='Generate a cross compilation file.') + parser.add_argument('--native', default=False, action='store_true', + help='Generate a native compilation file.') + parser.add_argument('--system', default=None, + help='Define system for cross compilation.') + parser.add_argument('--cpu', default=None, + help='Define cpu for cross compilation.') + parser.add_argument('--cpu-family', default=None, + help='Define cpu family for cross compilation.') + parser.add_argument('--endian', default='little', choices=['big', 'little'], + help='Define endianness for cross compilation.') + +class MachineInfo: + def __init__(self) -> None: + self.compilers: Dict[str, List[str]] = {} + self.binaries: Dict[str, List[str]] = {} + self.properties: Dict[str, Union[str, List[str]]] = {} + self.compile_args: Dict[str, List[str]] = {} + self.link_args: Dict[str, List[str]] = {} + + self.system: Optional[str] = None + self.cpu: Optional[str] = None + self.cpu_family: Optional[str] = None + self.endian: Optional[str] = None + +#parser = argparse.ArgumentParser(description='''Generate cross compilation definition file for the Meson build system. +# +#If you do not specify the --arch argument, Meson assumes that running +#plain 'dpkg-architecture' will return correct information for the +#host system. +# +#This script must be run in an environment where CPPFLAGS et al are set to the +#same values used in the actual compilation. +#''' +#) + +def locate_path(program: str) -> List[str]: + if os.path.isabs(program): + return [program] + for d in os.get_exec_path(): + f = os.path.join(d, program) + if os.access(f, os.X_OK): + return [f] + raise ValueError("%s not found on $PATH" % program) + +def write_args_line(ofile: TextIO, name: str, args: List[str]) -> None: + if len(args) == 0: + return + ostr = name + ' = [' + ostr += ', '.join("'" + i + "'" for i in args) + ostr += ']\n' + ofile.write(ostr) + +def get_args_from_envvars(infos: MachineInfo) -> None: + cppflags = shlex.split(os.environ.get('CPPFLAGS', '')) + cflags = shlex.split(os.environ.get('CFLAGS', '')) + cxxflags = shlex.split(os.environ.get('CXXFLAGS', '')) + objcflags = shlex.split(os.environ.get('OBJCFLAGS', '')) + objcxxflags = shlex.split(os.environ.get('OBJCXXFLAGS', '')) + ldflags = shlex.split(os.environ.get('LDFLAGS', '')) + + c_args = cppflags + cflags + cpp_args = cppflags + cxxflags + c_link_args = cflags + ldflags + cpp_link_args = cxxflags + ldflags + + objc_args = cppflags + objcflags + objcpp_args = cppflags + objcxxflags + objc_link_args = objcflags + ldflags + objcpp_link_args = objcxxflags + ldflags + + if c_args: + infos.compile_args['c'] = c_args + if c_link_args: + infos.link_args['c'] = c_link_args + if cpp_args: + infos.compile_args['cpp'] = cpp_args + if cpp_link_args: + infos.link_args['cpp'] = cpp_link_args + if objc_args: + infos.compile_args['objc'] = objc_args + if objc_link_args: + infos.link_args['objc'] = objc_link_args + if objcpp_args: + infos.compile_args['objcpp'] = objcpp_args + if objcpp_link_args: + infos.link_args['objcpp'] = objcpp_link_args + +cpu_family_map = dict(mips64el="mips64", + i686='x86') +cpu_map = dict(armhf="arm7hlf", + mips64el="mips64",) + +def deb_compiler_lookup(infos: MachineInfo, compilerstems: List[Tuple[str, str]], host_arch: str, gccsuffix: str) -> None: + for langname, stem in compilerstems: + compilername = f'{host_arch}-{stem}{gccsuffix}' + try: + p = locate_path(compilername) + infos.compilers[langname] = p + except ValueError: + pass + +def detect_cross_debianlike(options: Any) -> MachineInfo: + if options.debarch is None: + cmd = ['dpkg-architecture'] + else: + cmd = ['dpkg-architecture', '-a' + options.debarch] + output = subprocess.check_output(cmd, universal_newlines=True, + stderr=subprocess.DEVNULL) + data = {} + for line in output.split('\n'): + line = line.strip() + if line == '': + continue + k, v = line.split('=', 1) + data[k] = v + host_arch = data['DEB_HOST_GNU_TYPE'] + host_os = data['DEB_HOST_ARCH_OS'] + host_cpu_family = cpu_family_map.get(data['DEB_HOST_GNU_CPU'], + data['DEB_HOST_GNU_CPU']) + host_cpu = cpu_map.get(data['DEB_HOST_ARCH'], + data['DEB_HOST_ARCH']) + host_endian = data['DEB_HOST_ARCH_ENDIAN'] + + compilerstems = [('c', 'gcc'), + ('cpp', 'h++'), + ('objc', 'gobjc'), + ('objcpp', 'gobjc++')] + infos = MachineInfo() + deb_compiler_lookup(infos, compilerstems, host_arch, options.gccsuffix) + if len(infos.compilers) == 0: + print('Warning: no compilers were detected.') + infos.binaries['ar'] = locate_path("%s-ar" % host_arch) + infos.binaries['strip'] = locate_path("%s-strip" % host_arch) + infos.binaries['objcopy'] = locate_path("%s-objcopy" % host_arch) + infos.binaries['ld'] = locate_path("%s-ld" % host_arch) + try: + infos.binaries['pkgconfig'] = locate_path("%s-pkg-config" % host_arch) + except ValueError: + pass # pkg-config is optional + try: + infos.binaries['cups-config'] = locate_path("cups-config") + except ValueError: + pass + infos.system = host_os + infos.cpu_family = host_cpu_family + infos.cpu = host_cpu + infos.endian = host_endian + + get_args_from_envvars(infos) + return infos + +def write_machine_file(infos: MachineInfo, ofilename: str, write_system_info: bool) -> None: + tmpfilename = ofilename + '~' + with open(tmpfilename, 'w') as ofile: + ofile.write('[binaries]\n') + ofile.write('# Compilers\n') + for langname in sorted(infos.compilers.keys()): + compiler = infos.compilers[langname] + write_args_line(ofile, langname, compiler) + ofile.write('\n') + + ofile.write('# Other binaries\n') + for exename in sorted(infos.binaries.keys()): + exe = infos.binaries[exename] + write_args_line(ofile, exename, exe) + ofile.write('\n') + + ofile.write('[properties]\n') + all_langs = list(set(infos.compile_args.keys()).union(set(infos.link_args.keys()))) + all_langs.sort() + for lang in all_langs: + if lang in infos.compile_args: + write_args_line(ofile, lang + '_args', infos.compile_args[lang]) + if lang in infos.link_args: + write_args_line(ofile, lang + '_link_args', infos.link_args[lang]) + ofile.write('\n') + + if write_system_info: + ofile.write('[host_machine]\n') + ofile.write(f"cpu = '{infos.cpu}'\n") + ofile.write(f"cpu_family = '{infos.cpu_family}'\n") + ofile.write(f"endian = '{infos.endian}'\n") + ofile.write(f"system = '{infos.system}'\n") + os.replace(tmpfilename, ofilename) + +def detect_language_args_from_envvars(langname: str, envvar_suffix: str ='') -> Tuple[List[str], List[str]]: + ldflags = tuple(shlex.split(os.environ.get('LDFLAGS' + envvar_suffix, ''))) + compile_args = shlex.split(os.environ.get(UNIXY_ENVVARS_FLAGS[langname] + envvar_suffix, '')) + if langname in LANGS_USING_CPPFLAGS: + cppflags = tuple(shlex.split(os.environ.get('CPPFLAGS' + envvar_suffix, ''))) + lang_compile_args = list(cppflags) + compile_args + else: + lang_compile_args = compile_args + lang_link_args = list(ldflags) + compile_args + return (lang_compile_args, lang_link_args) + +def detect_compilers_from_envvars(envvar_suffix:str ='') -> MachineInfo: + infos = MachineInfo() + for langname, envvarname in UNIXY_ENVVARS_COMPILER.items(): + compilerstr = os.environ.get(envvarname + envvar_suffix) + if not compilerstr: + continue + compiler = shlex.split(compilerstr) + infos.compilers[langname] = compiler + lang_compile_args, lang_link_args = detect_language_args_from_envvars(langname, envvar_suffix) + if lang_compile_args: + infos.compile_args[langname] = lang_compile_args + if lang_link_args: + infos.link_args[langname] = lang_link_args + return infos + +def detect_binaries_from_envvars(infos: MachineInfo, envvar_suffix:str ='') -> None: + for binname, envvar_base in UNIXY_ENVVARS_TOOLS.items(): + envvar = envvar_base + envvar_suffix + binstr = os.environ.get(envvar) + if binstr: + infos.binaries[binname] = shlex.split(binstr) + +def detect_cross_system(infos: MachineInfo, options: Any) -> None: + for optname in ('system', 'cpu', 'cpu_family', 'endian'): + v = getattr(options, optname) + if not v: + mlog.error(f'Cross property "{optname}" missing, set it with --{optname.replace("_", "-")}.') + sys.exit(1) + setattr(infos, optname, v) + +def detect_cross_env(options: Any) -> MachineInfo: + if options.debarch: + print('Detecting cross environment via dpkg-reconfigure.') + infos = detect_cross_debianlike(options) + else: + print('Detecting cross environment via environment variables.') + infos = detect_compilers_from_envvars() + detect_cross_system(infos, options) + return infos + +def add_compiler_if_missing(infos: MachineInfo, langname: str, exe_names: List[str]) -> None: + if langname in infos.compilers: + return + for exe_name in exe_names: + lookup = shutil.which(exe_name) + if not lookup: + continue + compflags, linkflags = detect_language_args_from_envvars(langname) + infos.compilers[langname] = [lookup] + if compflags: + infos.compile_args[langname] = compflags + if linkflags: + infos.link_args[langname] = linkflags + return + +def detect_missing_native_compilers(infos: MachineInfo) -> None: + # Any per-platform special detection should go here. + for langname, exes in TYPICAL_UNIXY_COMPILER_NAMES.items(): + add_compiler_if_missing(infos, langname, exes) + +def detect_missing_native_binaries(infos: MachineInfo) -> None: + # Any per-platform special detection should go here. + for toolname in sorted(UNIXY_ENVVARS_TOOLS.keys()): + if toolname in infos.binaries: + continue + exe = shutil.which(toolname) + if exe: + infos.binaries[toolname] = [exe] + +def detect_native_env(options: Any) -> MachineInfo: + use_for_build = has_for_build() + if use_for_build: + mlog.log('Using FOR_BUILD envvars for detection') + esuffix = '_FOR_BUILD' + else: + mlog.log('Using regular envvars for detection.') + esuffix = '' + infos = detect_compilers_from_envvars(esuffix) + detect_missing_native_compilers(infos) + detect_binaries_from_envvars(infos, esuffix) + detect_missing_native_binaries(infos) + return infos + +def run(options: Any) -> None: + if options.cross and options.native: + sys.exit('You can only specify either --cross or --native, not both.') + if not options.cross and not options.native: + sys.exit('You must specify --cross or --native.') + mlog.notice('This functionality is experimental and subject to change.') + detect_cross = options.cross + if detect_cross: + infos = detect_cross_env(options) + write_system_info = True + else: + infos = detect_native_env(options) + write_system_info = False + write_machine_file(infos, options.outfile, write_system_info)