Add new env2cross command.

docs/markdown/snippets/env2cross.md

@@ -0,0 +1,40 @@
+## Experimental command to convert environments to cross files
+
+Meson has a new command `env2mfile` that can be used to convert
+"environment variable based" cross and native compilation environments
+to Meson machine files. This is especially convenient for e.g. distro
+packagers so they can easily generate unambiguous configuration files
+for packge building.
+
+As an example here's how you would generate a cross file that takes
+its settings from the `CC`, `CXX`, `CFLAGS` etc environment variables.
+
+    meson env2mfile --cross --system=baremetal --cpu=armv7 --cpu-family=arm -o armcross.txt
+
+The command also has support for generating Debian build files using
+system introspection:
+
+    meson env2mfile --cross --debarch armhf -o debarmhf_cross.txt
+
+Note how you don't need to specify any system details, the command
+gets them transparently via `dpkg-architecture`.
+
+Creating a native file is done in the same way:
+
+    meson env2mfile --native -o current_system.txt
+
+This system will detect if the `_FOR_BUILD` environment variables are
+enabled and then uses them as needed.
+
+With this you should be able to convert any envvar-based cross build
+setup to cross and native files and then use those. Thit means, among
+other things, that you can then run your compilations from any shell,
+not just the special one that has all the environment variables set.
+
+As this functionality is still a bit in flux, the specific behaviour
+and command line arguments to use are subject to change. Because of
+this the main documentation has not yet been updated.
+
+Please try this for your use cases and report to us if it is working.
+Patches to make the autodetection work on other distros and platforms
+are also welcome.

mesonbuild/mesonmain.py

@@ -30,7 +30,7 @@ from . import mconf, mdist, minit, minstall, mintro, msetup, mtest, rewriter, ms
 from .mesonlib import MesonException, MesonBugException
 from .environment import detect_msys2_arch
 from .wrap import wraptool
-
+from .scripts import env2mfile
 
 # Note: when adding arguments, please also add them to the completion
 # scripts in $MESONSRC/data/shell-completions/
@@ -70,6 +70,8 @@ class CommandLineParser:
                          help_msg='Build the project')
         self.add_command('devenv', mdevenv.add_arguments, mdevenv.run,
                          help_msg='Run commands in developer environment')
+        self.add_command('env2mfile', env2mfile.add_arguments, env2mfile.run,
+                         help_msg='Convert current environment to a cross or native file')
 
         # Hidden commands
         self.add_command('runpython', self.add_runpython_arguments, self.run_runpython_command,

mesonbuild/scripts/env2mfile.py

@@ -0,0 +1,368 @@
+# Copyright 2022 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys, os, subprocess, shutil
+import shlex
+
+import argparse
+from typing import TextIO, Dict, List, Union, Tuple, Any, Optional
+
+from .. import mlog
+
+UNIXY_ENVVARS_COMPILER = {'c': 'CC',
+                          'cpp': 'CXX',
+                          'objc': 'OBJCC',
+                          'objcpp': 'OBJCXX',
+                          'fortran': 'FC',
+                          'rust': 'RUSTC',
+                          'vala': 'VALAC',
+                          'cs': 'CSC',
+                          }
+
+UNIXY_ENVVARS_TOOLS = {'ar': 'AR',
+                       'strip': 'STRIP',
+                       'windres': 'WINDRES',
+                       'pkgconfig': 'PKG_CONFIG',
+                       'vapigen': 'VAPIGEN',
+                       'cmake': 'CMAKE',
+                       'qmake': 'QMAKE',
+                       }
+
+UNIXY_ENVVARS_FLAGS = {'c': 'CFLAGS',
+                       'cpp': 'CXXFLAGS',
+                       'objc': 'OBJCFLAGS',
+                       'objcpp': 'OBJCXXFLAGS',
+                       'fortran': 'FFLAGS',
+                       'rust': 'RUSTFLAGS',
+                       'vala': 'VALAFLAGS',
+                       'cs': 'CSFLAGS', # This one might not be standard.
+                       }
+
+TYPICAL_UNIXY_COMPILER_NAMES = {'c': ['cc', 'gcc', 'clang'],
+                                'cpp': ['c++', 'g++', 'clang++'],
+                                'objc': ['objc', 'clang'],
+                                'objcpp': ['objcpp', 'clang++'],
+                                'fortran': ['gfortran'],
+                                }
+
+LANGS_USING_CPPFLAGS = {'c', 'cpp', 'objc', 'objcxx'}
+
+def has_for_build() -> bool:
+    for cenv in UNIXY_ENVVARS_COMPILER.values():
+        if os.environ.get(cenv + '_FOR_BUILD'):
+            return True
+    return False
+
+def add_arguments(parser: 'argparse.ArgumentParser') -> None:
+    parser.add_argument('--debarch', default=None,
+                        help='The dpkg architecture to generate.')
+    parser.add_argument('--gccsuffix', default="",
+                        help='A particular gcc version suffix if necessary.')
+    parser.add_argument('-o', required=True, dest='outfile',
+                        help='The output file.')
+    parser.add_argument('--cross', default=False, action='store_true',
+                        help='Generate a cross compilation file.')
+    parser.add_argument('--native', default=False, action='store_true',
+                        help='Generate a native compilation file.')
+    parser.add_argument('--system', default=None,
+                        help='Define system for cross compilation.')
+    parser.add_argument('--cpu', default=None,
+                        help='Define cpu for cross compilation.')
+    parser.add_argument('--cpu-family', default=None,
+                        help='Define cpu family for cross compilation.')
+    parser.add_argument('--endian', default='little', choices=['big', 'little'],
+                        help='Define endianness for cross compilation.')
+
+class MachineInfo:
+    def __init__(self) -> None:
+        self.compilers: Dict[str, List[str]] = {}
+        self.binaries: Dict[str, List[str]] = {}
+        self.properties: Dict[str, Union[str, List[str]]] = {}
+        self.compile_args: Dict[str, List[str]] = {}
+        self.link_args: Dict[str, List[str]] = {}
+
+        self.system: Optional[str] = None
+        self.cpu: Optional[str] = None
+        self.cpu_family: Optional[str] = None
+        self.endian: Optional[str] = None
+
+#parser = argparse.ArgumentParser(description='''Generate cross compilation definition file for the Meson build system.
+#
+#If you do not specify the --arch argument, Meson assumes that running
+#plain 'dpkg-architecture' will return correct information for the
+#host system.
+#
+#This script must be run in an environment where CPPFLAGS et al are set to the
+#same values used in the actual compilation.
+#'''
+#)
+
+def locate_path(program: str) -> List[str]:
+    if os.path.isabs(program):
+        return [program]
+    for d in os.get_exec_path():
+        f = os.path.join(d, program)
+        if os.access(f, os.X_OK):
+            return [f]
+    raise ValueError("%s not found on $PATH" % program)
+
+def write_args_line(ofile: TextIO, name: str, args: List[str]) -> None:
+    if len(args) == 0:
+        return
+    ostr = name + ' = ['
+    ostr += ', '.join("'" + i + "'" for i in args)
+    ostr += ']\n'
+    ofile.write(ostr)
+
+def get_args_from_envvars(infos: MachineInfo) -> None:
+    cppflags = shlex.split(os.environ.get('CPPFLAGS', ''))
+    cflags = shlex.split(os.environ.get('CFLAGS', ''))
+    cxxflags = shlex.split(os.environ.get('CXXFLAGS', ''))
+    objcflags = shlex.split(os.environ.get('OBJCFLAGS', ''))
+    objcxxflags = shlex.split(os.environ.get('OBJCXXFLAGS', ''))
+    ldflags = shlex.split(os.environ.get('LDFLAGS', ''))
+
+    c_args = cppflags + cflags
+    cpp_args = cppflags + cxxflags
+    c_link_args = cflags + ldflags
+    cpp_link_args = cxxflags + ldflags
+
+    objc_args = cppflags + objcflags
+    objcpp_args = cppflags + objcxxflags
+    objc_link_args = objcflags + ldflags
+    objcpp_link_args = objcxxflags + ldflags
+
+    if c_args:
+        infos.compile_args['c'] = c_args
+    if c_link_args:
+        infos.link_args['c'] = c_link_args
+    if cpp_args:
+        infos.compile_args['cpp'] = cpp_args
+    if cpp_link_args:
+        infos.link_args['cpp'] = cpp_link_args
+    if objc_args:
+        infos.compile_args['objc'] = objc_args
+    if objc_link_args:
+        infos.link_args['objc'] = objc_link_args
+    if objcpp_args:
+        infos.compile_args['objcpp'] = objcpp_args
+    if objcpp_link_args:
+        infos.link_args['objcpp'] = objcpp_link_args
+
+cpu_family_map = dict(mips64el="mips64",
+                      i686='x86')
+cpu_map = dict(armhf="arm7hlf",
+               mips64el="mips64",)
+
+def deb_compiler_lookup(infos: MachineInfo, compilerstems: List[Tuple[str, str]], host_arch: str, gccsuffix: str) -> None:
+    for langname, stem in compilerstems:
+        compilername = f'{host_arch}-{stem}{gccsuffix}'
+        try:
+            p = locate_path(compilername)
+            infos.compilers[langname] = p
+        except ValueError:
+            pass
+
+def detect_cross_debianlike(options: Any) -> MachineInfo:
+    if options.debarch is None:
+        cmd = ['dpkg-architecture']
+    else:
+        cmd = ['dpkg-architecture', '-a' + options.debarch]
+    output = subprocess.check_output(cmd, universal_newlines=True,
+                                     stderr=subprocess.DEVNULL)
+    data = {}
+    for line in output.split('\n'):
+        line = line.strip()
+        if line == '':
+            continue
+        k, v = line.split('=', 1)
+        data[k] = v
+    host_arch = data['DEB_HOST_GNU_TYPE']
+    host_os = data['DEB_HOST_ARCH_OS']
+    host_cpu_family = cpu_family_map.get(data['DEB_HOST_GNU_CPU'],
+                                         data['DEB_HOST_GNU_CPU'])
+    host_cpu = cpu_map.get(data['DEB_HOST_ARCH'],
+                           data['DEB_HOST_ARCH'])
+    host_endian = data['DEB_HOST_ARCH_ENDIAN']
+
+    compilerstems = [('c', 'gcc'),
+                     ('cpp', 'h++'),
+                     ('objc', 'gobjc'),
+                     ('objcpp', 'gobjc++')]
+    infos = MachineInfo()
+    deb_compiler_lookup(infos, compilerstems, host_arch, options.gccsuffix)
+    if len(infos.compilers) == 0:
+        print('Warning: no compilers were detected.')
+    infos.binaries['ar'] = locate_path("%s-ar" % host_arch)
+    infos.binaries['strip'] = locate_path("%s-strip" % host_arch)
+    infos.binaries['objcopy'] = locate_path("%s-objcopy" % host_arch)
+    infos.binaries['ld'] = locate_path("%s-ld" % host_arch)
+    try:
+        infos.binaries['pkgconfig'] =  locate_path("%s-pkg-config" % host_arch)
+    except ValueError:
+        pass # pkg-config is optional
+    try:
+        infos.binaries['cups-config'] = locate_path("cups-config")
+    except ValueError:
+        pass
+    infos.system = host_os
+    infos.cpu_family = host_cpu_family
+    infos.cpu = host_cpu
+    infos.endian = host_endian
+
+    get_args_from_envvars(infos)
+    return infos
+
+def write_machine_file(infos: MachineInfo, ofilename: str, write_system_info: bool) -> None:
+    tmpfilename = ofilename + '~'
+    with open(tmpfilename, 'w') as ofile:
+        ofile.write('[binaries]\n')
+        ofile.write('# Compilers\n')
+        for langname in sorted(infos.compilers.keys()):
+            compiler = infos.compilers[langname]
+            write_args_line(ofile, langname, compiler)
+        ofile.write('\n')
+
+        ofile.write('# Other binaries\n')
+        for exename in sorted(infos.binaries.keys()):
+            exe = infos.binaries[exename]
+            write_args_line(ofile, exename, exe)
+        ofile.write('\n')
+
+        ofile.write('[properties]\n')
+        all_langs = list(set(infos.compile_args.keys()).union(set(infos.link_args.keys())))
+        all_langs.sort()
+        for lang in all_langs:
+            if lang in infos.compile_args:
+                write_args_line(ofile, lang + '_args', infos.compile_args[lang])
+            if lang in infos.link_args:
+                write_args_line(ofile, lang + '_link_args', infos.link_args[lang])
+        ofile.write('\n')
+
+        if write_system_info:
+            ofile.write('[host_machine]\n')
+            ofile.write(f"cpu = '{infos.cpu}'\n")
+            ofile.write(f"cpu_family = '{infos.cpu_family}'\n")
+            ofile.write(f"endian = '{infos.endian}'\n")
+            ofile.write(f"system = '{infos.system}'\n")
+    os.replace(tmpfilename, ofilename)
+
+def detect_language_args_from_envvars(langname: str, envvar_suffix: str ='') -> Tuple[List[str], List[str]]:
+    ldflags = tuple(shlex.split(os.environ.get('LDFLAGS' + envvar_suffix, '')))
+    compile_args = shlex.split(os.environ.get(UNIXY_ENVVARS_FLAGS[langname] + envvar_suffix, ''))
+    if langname in LANGS_USING_CPPFLAGS:
+        cppflags = tuple(shlex.split(os.environ.get('CPPFLAGS' + envvar_suffix, '')))
+        lang_compile_args = list(cppflags) + compile_args
+    else:
+        lang_compile_args = compile_args
+    lang_link_args = list(ldflags) + compile_args
+    return (lang_compile_args, lang_link_args)
+
+def detect_compilers_from_envvars(envvar_suffix:str ='') -> MachineInfo:
+    infos = MachineInfo()
+    for langname, envvarname in UNIXY_ENVVARS_COMPILER.items():
+        compilerstr = os.environ.get(envvarname + envvar_suffix)
+        if not compilerstr:
+            continue
+        compiler = shlex.split(compilerstr)
+        infos.compilers[langname] = compiler
+        lang_compile_args, lang_link_args = detect_language_args_from_envvars(langname, envvar_suffix)
+        if lang_compile_args:
+            infos.compile_args[langname] = lang_compile_args
+        if lang_link_args:
+            infos.link_args[langname] = lang_link_args
+    return infos
+
+def detect_binaries_from_envvars(infos: MachineInfo, envvar_suffix:str ='') -> None:
+    for binname, envvar_base in UNIXY_ENVVARS_TOOLS.items():
+        envvar = envvar_base + envvar_suffix
+        binstr = os.environ.get(envvar)
+        if binstr:
+            infos.binaries[binname] = shlex.split(binstr)
+
+def detect_cross_system(infos: MachineInfo, options: Any) -> None:
+    for optname in ('system', 'cpu', 'cpu_family', 'endian'):
+        v = getattr(options, optname)
+        if not v:
+            mlog.error(f'Cross property "{optname}" missing, set it with --{optname.replace("_", "-")}.')
+            sys.exit(1)
+        setattr(infos, optname, v)
+
+def detect_cross_env(options: Any) -> MachineInfo:
+    if options.debarch:
+        print('Detecting cross environment via dpkg-reconfigure.')
+        infos = detect_cross_debianlike(options)
+    else:
+        print('Detecting cross environment via environment variables.')
+        infos = detect_compilers_from_envvars()
+        detect_cross_system(infos, options)
+    return infos
+
+def add_compiler_if_missing(infos: MachineInfo, langname: str, exe_names: List[str]) -> None:
+    if langname in infos.compilers:
+        return
+    for exe_name in exe_names:
+        lookup = shutil.which(exe_name)
+        if not lookup:
+            continue
+        compflags, linkflags = detect_language_args_from_envvars(langname)
+        infos.compilers[langname] = [lookup]
+        if compflags:
+            infos.compile_args[langname] = compflags
+        if linkflags:
+            infos.link_args[langname] = linkflags
+        return
+
+def detect_missing_native_compilers(infos: MachineInfo) -> None:
+    # Any per-platform special detection should go here.
+    for langname, exes in TYPICAL_UNIXY_COMPILER_NAMES.items():
+        add_compiler_if_missing(infos, langname, exes)
+
+def detect_missing_native_binaries(infos: MachineInfo) -> None:
+    # Any per-platform special detection should go here.
+    for toolname in sorted(UNIXY_ENVVARS_TOOLS.keys()):
+        if toolname in infos.binaries:
+            continue
+        exe = shutil.which(toolname)
+        if exe:
+            infos.binaries[toolname] = [exe]
+
+def detect_native_env(options: Any) -> MachineInfo:
+    use_for_build = has_for_build()
+    if use_for_build:
+        mlog.log('Using FOR_BUILD envvars for detection')
+        esuffix = '_FOR_BUILD'
+    else:
+        mlog.log('Using regular envvars for detection.')
+        esuffix = ''
+    infos = detect_compilers_from_envvars(esuffix)
+    detect_missing_native_compilers(infos)
+    detect_binaries_from_envvars(infos, esuffix)
+    detect_missing_native_binaries(infos)
+    return infos
+
+def run(options: Any) -> None:
+    if options.cross and options.native:
+        sys.exit('You can only specify either --cross or --native, not both.')
+    if not options.cross and not options.native:
+        sys.exit('You must specify --cross or --native.')
+    mlog.notice('This functionality is experimental and subject to change.')
+    detect_cross = options.cross
+    if detect_cross:
+        infos = detect_cross_env(options)
+        write_system_info = True
+    else:
+        infos = detect_native_env(options)
+        write_system_info = False
+    write_machine_file(infos, options.outfile, write_system_info)