Put machine file and cmd line parsing in Environment

This creates a full set of option in environment that mirror those in
coredata, this mirroring of the coredata structure is convenient because
lookups int env (such as when initializing compilers) becomes a straight
dict lookup, with no list iteration. It also means that all of the
command line and machine files are read and stored in the correct order
before they're ever accessed, simplifying the logic of using them.

mesonbuild/ast/introspection.py

@@ -120,7 +120,7 @@ class IntrospectionInterpreter(AstInterpreter):
                         self.do_subproject(i)
 
         self.coredata.init_backend_options(self.backend)
-        options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')}
+        options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')}
 
         self.coredata.set_options(options)
         self.func_add_languages(None, proj_langs, None)

mesonbuild/coredata.py

@@ -20,10 +20,8 @@ from pathlib import PurePath
 from collections import OrderedDict, defaultdict
 from .mesonlib import (
     MesonException, EnvironmentException, MachineChoice, PerMachine,
-    OrderedSet, default_libdir, default_libexecdir, default_prefix,
+    default_libdir, default_libexecdir, default_prefix, split_args
-    split_args
 )
-from .envconfig import get_env_var_pair
 from .wrap import WrapMode
 import ast
 import argparse
@@ -741,117 +739,54 @@ class CoreData:
         if not self.is_cross_build():
             self.copy_build_options_from_regular_ones()
 
-    def set_default_options(self, default_options: T.Mapping[str, str], subproject: str, env: 'Environment') -> None:
+    def set_default_options(self, default_options: 'T.OrderedDict[str, str]', subproject: str, env: 'Environment') -> None:
-        # Warn if the user is using two different ways of setting build-type
+        def make_key(key: str) -> str:
-        # options that override each other
+            if subproject:
-        if 'buildtype' in env.cmd_line_options and \
+                return '{}:{}'.format(subproject, key)
-           ('optimization' in env.cmd_line_options or 'debug' in env.cmd_line_options):
+            return key
-            mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. '
-                         'Using both is redundant since they override each other. '
-                         'See: https://mesonbuild.com/Builtin-options.html#build-type-options')
-
-        cmd_line_options = OrderedDict()
-
-        from . import optinterpreter
-        from .compilers import all_languages
-        if not subproject:
-            # Set default options as if they were passed to the command line.
-            # Subprojects can only define default for user options and not yielding
-            # builtin option.
-            for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items()):
-                cmd_line_options[k] = v
-
-            # compiler options are always per-machine, but not per sub-project
-            if '' in env.meson_options.build:
-                for lang in all_languages:
-                    prefix = '{}_'.format(lang)
-                    for k in env.meson_options.build['']:
-                        if k.startswith(prefix):
-                            cmd_line_options['build.{}'.format(k)] = env.meson_options.build[subproject][k]
-        else:
-            # If the subproject options comes from a machine file, then we need to
-            # set the option as subproject:option
-            for k, v in chain(default_options.items(), env.meson_options.host.get('', {}).items(),
-                              env.meson_options.host.get(subproject, {}).items()):
-                if (k not in builtin_options or builtin_options[k].yielding) \
-                        and optinterpreter.is_invalid_name(k, log=False):
-                    continue
-                cmd_line_options['{}:{}'.format(subproject, k)] = v
-        cmd_line_options.update(env.cmd_line_options)
-        env.cmd_line_options = cmd_line_options
 
         options = OrderedDict()
 
-        # load the values for user options out of the appropriate machine file,
+        # TODO: validate these
-        # then overload the command line
+        from .compilers import all_languages, base_options
-        for k, v in env.user_options.get(subproject, {}).items():
+        lang_prefixes = tuple('{}_'.format(l) for l in all_languages)
-            if subproject:
+        # split arguments that can be set now, and those that cannot so they
-                k = '{}:{}'.format(subproject, k)
+        # can be set later, when they've been initialized.
-            options[k] = v
+        for k, v in default_options.items():
-
+            if k.startswith(lang_prefixes):
-        # Report that [properties]c_args
+                lang, key = k.split('_', 1)
-        for lang in all_languages:
+                for machine in MachineChoice:
-            for args in ['{}_args'.format(lang), '{}_link_args'.format(lang)]:
+                    if key not in env.compiler_options[machine][lang]:
-                msg = ('{} in the [properties] section of the machine file is deprecated, '
+                         env.compiler_options[machine][lang][key] = v
-                       'use the [built-in options] section.')
+            elif k in base_options:
-                if args in env.properties.host or args in env.properties.build:
+                if not subproject and k not in env.base_options:
-                    mlog.deprecation(msg.format(args))
+                    env.base_options[k] = v
-
-        # Currently we don't support any options that are both per-subproject
-        # and per-machine, but when we do this will need to account for that.
-        # For cross builds we need to get the build specifc options
-        if env.meson_options.host != env.meson_options.build and subproject in env.meson_options.build:
-            if subproject:
-                template = '{s}:build.{k}'
             else:
-                template = 'build.{k}'
+                options[make_key(k)] = v
-            for k in builtin_options_per_machine.keys():
+
-                if k in env.meson_options.build[subproject]:
+        for k, v in chain(env.meson_options.host.get('', {}).items(),
-                    options[template.format(s=subproject, k=k)] = env.meson_options.build[subproject][k]
+                          env.meson_options.host.get(subproject, {}).items()):
-
+            options[make_key(k)] = v
-        # Some options default to environment variables if they are
+
-        # unset, set those now. These will either be overwritten
+        for k, v in chain(env.meson_options.build.get('', {}).items(),
-        # below, or they won't. These should only be set on the first run.
+                          env.meson_options.build.get(subproject, {}).items()):
-        for for_machine in MachineChoice:
+            if k in builtin_options_per_machine:
-            p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), 'PKG_CONFIG_PATH')
+                options[make_key('build.{}'.format(k))] = v
-            if p_env_pair is not None:
+
-                p_env_var, p_env = p_env_pair
+        options.update({make_key(k): v for k, v in env.user_options.get(subproject, {}).items()})
-
+
-                # PKG_CONFIG_PATH may contain duplicates, which must be
+        # Some options (namely the compiler options) are not preasant in
-                # removed, else a duplicates-in-array-option warning arises.
+        # coredata until the compiler is fully initialized. As such, we need to
-                p_list = list(OrderedSet(p_env.split(':')))
+        # put those options into env.meson_options, only if they're not already
-
+        # in there, as the machine files and command line have precendence.
-                key = 'pkg_config_path'
+        for k, v in default_options.items():
-                if for_machine == MachineChoice.BUILD:
+            if k in builtin_options and not builtin_options[k].yielding:
-                    key = 'build.' + key
+                continue
-
+            for machine in MachineChoice:
-                if env.first_invocation:
+                if machine is MachineChoice.BUILD and not self.is_cross_build():
-                    options[key] = p_list
-                elif options.get(key, []) != p_list:
-                    mlog.warning(
-                        p_env_var +
-                        ' environment variable has changed '
-                        'between configurations, meson ignores this. '
-                        'Use -Dpkg_config_path to change pkg-config search '
-                        'path instead.'
-                    )
-
-        def remove_prefix(text, prefix):
-            if text.startswith(prefix):
-                return text[len(prefix):]
-            return text
-
-        for k, v in cmd_line_options.items():
-            if subproject:
-                if not k.startswith(subproject + ':'):
-                    continue
-            elif k not in builtin_options.keys() \
-                    and remove_prefix(k, 'build.') not in builtin_options_per_machine.keys():
-                if ':' in k:
-                    continue
-                if optinterpreter.is_invalid_name(k, log=False):
                     continue
-            options[k] = v
+                if k not in env.meson_options[machine][subproject]:
+                    env.meson_options[machine][subproject][k] = v
 
         self.set_options(options, subproject=subproject)
 
@@ -867,24 +802,19 @@ class CoreData:
                 env.is_cross_build(),
                 env.properties[for_machine]).items():
             # prefixed compiler options affect just this machine
-            opt_prefix = for_machine.get_prefix()
+            if k in env.compiler_options[for_machine].get(lang, {}):
-            user_k = opt_prefix + lang + '_' + k
+                o.set_value(env.compiler_options[for_machine][lang][k])
-            if user_k in env.cmd_line_options:
-                o.set_value(env.cmd_line_options[user_k])
             self.compiler_options[for_machine][lang].setdefault(k, o)
 
-    def process_new_compiler(self, lang: str, comp: T.Type['Compiler'], env: 'Environment') -> None:
+    def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None:
         from . import compilers
 
         self.compilers[comp.for_machine][lang] = comp
-        enabled_opts = []
 
         for k, o in comp.get_options().items():
             # prefixed compiler options affect just this machine
-            opt_prefix = comp.for_machine.get_prefix()
+            if k in env.compiler_options[comp.for_machine].get(lang, {}):
-            user_k = opt_prefix + lang + '_' + k
+                o.set_value(env.compiler_options[comp.for_machine][lang][k])
-            if user_k in env.cmd_line_options:
-                o.set_value(env.cmd_line_options[user_k])
             self.compiler_options[comp.for_machine][lang].setdefault(k, o)
 
         enabled_opts = []
@@ -892,8 +822,8 @@ class CoreData:
             if optname in self.base_options:
                 continue
             oobj = compilers.base_options[optname]
-            if optname in env.cmd_line_options:
+            if optname in env.base_options:
-                oobj.set_value(env.cmd_line_options[optname])
+                oobj.set_value(env.base_options[optname])
                 enabled_opts.append(optname)
             self.base_options[optname] = oobj
         self.emit_base_options_warnings(enabled_opts)

mesonbuild/environment.py

@@ -16,6 +16,7 @@ import os, platform, re, sys, shutil, subprocess
 import tempfile
 import shlex
 import typing as T
+import collections
 
 from . import coredata
 from .linkers import ArLinker, ArmarLinker, VisualStudioLinker, DLinker, CcrxLinker, Xc16Linker, C2000Linker, IntelVisualStudioLinker
@@ -28,11 +29,13 @@ from . import mlog
 
 from .envconfig import (
     BinaryTable, MachineInfo,
-    Properties, known_cpu_families,
+    Properties, known_cpu_families, get_env_var_pair,
 )
 from . import compilers
 from .compilers import (
     Compiler,
+    all_languages,
+    base_options,
     is_assembly,
     is_header,
     is_library,
@@ -549,10 +552,10 @@ class Environment:
         properties = PerMachineDefaultable()
 
         # We only need one of these as project options are not per machine
-        user_options = {}
+        user_options = collections.defaultdict(dict)  # type: T.DefaultDict[str, T.Dict[str, object]]
 
         # meson builtin options, as passed through cross or native files
-        meson_options = PerMachineDefaultable()
+        meson_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
 
         ## Setup build machine defaults
 
@@ -564,6 +567,13 @@ class Environment:
         binaries.build = BinaryTable()
         properties.build = Properties()
 
+        # meson base options
+        _base_options = {}  # type: T.Dict[str, object]
+
+        # Per language compiler arguments
+        compiler_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
+        compiler_options.build = collections.defaultdict(dict)
+
         ## Read in native file(s) to override build machine configuration
 
         def load_options(tag: str, store: T.Dict[str, T.Any]) -> None:
@@ -573,7 +583,44 @@ class Environment:
                         project = section.split(':')[0]
                     else:
                         project = ''
-                    store[project] = config.get(section, {})
+                    store[project].update(config.get(section, {}))
+
+        def split_base_options(mopts: T.DefaultDict[str, T.Dict[str, object]]) -> None:
+            for k, v in list(mopts.get('', {}).items()):
+                if k in base_options:
+                    _base_options[k] = v
+                    del mopts[k]
+
+        lang_prefixes = tuple('{}_'.format(l) for l in all_languages)
+        def split_compiler_options(mopts: T.DefaultDict[str, T.Dict[str, object]], machine: MachineChoice) -> None:
+            for k, v in list(mopts.get('', {}).items()):
+                if k.startswith(lang_prefixes):
+                    lang, key = k.split('_', 1)
+                    if compiler_options[machine] is None:
+                        compiler_options[machine] = collections.defaultdict(dict)
+                    if lang not in compiler_options[machine]:
+                        compiler_options[machine][lang] = collections.defaultdict(dict)
+                    compiler_options[machine][lang][key] = v
+                    del mopts[''][k]
+
+        def move_compiler_options(properties: Properties, compopts: T.Dict[str, T.DefaultDict[str, object]]) -> None:
+            for k, v in properties.properties.copy().items():
+                for lang in all_languages:
+                    if k == '{}_args'.format(lang):
+                        if 'args' not in compopts[lang]:
+                            compopts[lang]['args'] = v
+                        else:
+                            mlog.warning('Ignoring {}_args in [properties] section for those in the [built-in options]'.format(lang))
+                    elif k == '{}_link_args'.format(lang):
+                        if 'link_args' not in compopts[lang]:
+                            compopts[lang]['link_args'] = v
+                        else:
+                            mlog.warning('Ignoring {}_link_args in [properties] section in favor of the [built-in options] section.')
+                    else:
+                        continue
+                    mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k))
+                    del properties.properties[k]
+                    break
 
         if self.coredata.config_files is not None:
             config = coredata.parse_machine_files(self.coredata.config_files)
@@ -584,11 +631,15 @@ class Environment:
             # the native values if we're doing a cross build
             if not self.coredata.cross_files:
                 load_options('project options', user_options)
-            meson_options.build = {}
+            meson_options.build = collections.defaultdict(dict)
             if config.get('paths') is not None:
                 mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
                 load_options('paths', meson_options.build)
             load_options('built-in options', meson_options.build)
+            if not self.coredata.cross_files:
+                split_base_options(meson_options.build)
+            split_compiler_options(meson_options.build, MachineChoice.BUILD)
+            move_compiler_options(properties.build, compiler_options.build)
 
         ## Read in cross file(s) to override host machine configuration
 
@@ -601,11 +652,15 @@ class Environment:
             if 'target_machine' in config:
                 machines.target = MachineInfo.from_literal(config['target_machine'])
             load_options('project options', user_options)
-            meson_options.host = {}
+            meson_options.host = collections.defaultdict(dict)
+            compiler_options.host = collections.defaultdict(dict)
             if config.get('paths') is not None:
                 mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
                 load_options('paths', meson_options.host)
             load_options('built-in options', meson_options.host)
+            split_base_options(meson_options.host)
+            split_compiler_options(meson_options.host, MachineChoice.HOST)
+            move_compiler_options(properties.host, compiler_options.host)
 
         ## "freeze" now initialized configuration, and "save" to the class.
 
@@ -614,6 +669,67 @@ class Environment:
         self.properties = properties.default_missing()
         self.user_options = user_options
         self.meson_options = meson_options.default_missing()
+        self.base_options = _base_options
+        self.compiler_options = compiler_options.default_missing()
+
+        # Some options default to environment variables if they are
+        # unset, set those now.
+
+        for for_machine in MachineChoice:
+            p_env_pair = get_env_var_pair(for_machine, self.coredata.is_cross_build(), 'PKG_CONFIG_PATH')
+            if p_env_pair is not None:
+                p_env_var, p_env = p_env_pair
+
+                # PKG_CONFIG_PATH may contain duplicates, which must be
+                # removed, else a duplicates-in-array-option warning arises.
+                p_list = list(mesonlib.OrderedSet(p_env.split(':')))
+
+                key = 'pkg_config_path'
+
+                if self.first_invocation:
+                    # Environment variables override config
+                    self.meson_options[for_machine][''][key] = p_list
+                elif self.meson_options[for_machine][''].get(key, []) != p_list:
+                    mlog.warning(
+                        p_env_var,
+                        'environment variable does not match configured',
+                        'between configurations, meson ignores this.',
+                        'Use -Dpkg_config_path to change pkg-config search',
+                        'path instead.'
+                    )
+
+        # Read in command line and populate options
+        # TODO: validate all of this
+        all_builtins = set(coredata.builtin_options) | set(coredata.builtin_options_per_machine) | set(coredata.builtin_dir_noprefix_options)
+        for k, v in options.cmd_line_options.items():
+            try:
+                subproject, k = k.split(':')
+            except ValueError:
+                subproject = ''
+            if k in base_options:
+                self.base_options[k] = v
+            elif k.startswith(lang_prefixes):
+                lang, key = k.split('_', 1)
+                self.compiler_options.host[lang][key] = v
+            elif k in all_builtins or k.startswith('backend_'):
+                self.meson_options.host[subproject][k] = v
+            elif k.startswith('build.'):
+                k = k.lstrip('build.')
+                if k in coredata.builtin_options_per_machine:
+                    if self.meson_options.build is None:
+                        self.meson_options.build = collections.defaultdict(dict)
+                    self.meson_options.build[subproject][k] = v
+            else:
+                assert not k.startswith('build.')
+                self.user_options[subproject][k] = v
+
+        # Warn if the user is using two different ways of setting build-type
+        # options that override each other
+        if meson_options.build and 'buildtype' in meson_options.build[''] and \
+           ('optimization' in meson_options.build[''] or 'debug' in meson_options.build['']):
+            mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. '
+                         'Using both is redundant since they override each other. '
+                         'See: https://mesonbuild.com/Builtin-options.html#build-type-options')
 
         exe_wrapper = self.lookup_binary_entry(MachineChoice.HOST, 'exe_wrapper')
         if exe_wrapper is not None:
@@ -622,8 +738,6 @@ class Environment:
         else:
             self.exe_wrapper = None
 
-        self.cmd_line_options = options.cmd_line_options.copy()
-
         # List of potential compilers.
         if mesonlib.is_windows():
             # Intel C and C++ compiler is icl on Windows, but icc and icpc elsewhere.

mesonbuild/interpreter.py

@@ -2946,6 +2946,7 @@ external dependencies (including libraries) must go to "dependencies".''')
         if self.is_subproject():
             optname = self.subproject + ':' + optname
 
+
         for opts in [
                 self.coredata.base_options, compilers.base_options, self.coredata.builtins,
                 dict(self.coredata.get_prefixed_options_per_machine(self.coredata.builtins_per_machine)),
@@ -3031,8 +3032,9 @@ external dependencies (including libraries) must go to "dependencies".''')
         if self.environment.first_invocation:
             self.coredata.init_backend_options(backend)
 
-        options = {k: v for k, v in self.environment.cmd_line_options.items() if k.startswith('backend_')}
+        if '' in self.environment.meson_options.host:
-        self.coredata.set_options(options)
+            options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')}
+            self.coredata.set_options(options)
 
     @stringArgs
     @permittedKwargs(permitted_kwargs['project'])
@@ -3065,7 +3067,7 @@ external dependencies (including libraries) must go to "dependencies".''')
         self.project_default_options = mesonlib.stringlistify(kwargs.get('default_options', []))
         self.project_default_options = coredata.create_options_dict(self.project_default_options)
         if self.environment.first_invocation:
-            default_options = self.project_default_options
+            default_options = self.project_default_options.copy()
             default_options.update(self.default_project_options)
             self.coredata.init_builtins(self.subproject)
         else:

run_unittests.py

@@ -8030,7 +8030,7 @@ class NativeFileTests(BasePlatformTests):
         for opt, value in [('testoption', 'some other val'), ('other_one', True),
                            ('combo_opt', 'one'), ('array_opt', ['two']),
                            ('integer_opt', 0)]:
-            config = self.helper_create_native_file({'project options': {'sub:{}'.format(opt): value}})
+            config = self.helper_create_native_file({'sub:project options': {opt: value}})
             with self.assertRaises(subprocess.CalledProcessError) as cm:
                 self.init(testcase, extra_args=['--native-file', config])
                 self.assertRegex(cm.exception.stdout, r'Incorrect value to [a-z]+ option')
@@ -8076,6 +8076,19 @@ class NativeFileTests(BasePlatformTests):
         else:
             self.fail('Did not find werror in build options?')
 
+    def test_builtin_options_env_overrides_conf(self):
+        testcase = os.path.join(self.common_test_dir, '2 cpp')
+        config = self.helper_create_native_file({'built-in options': {'pkg_config_path': '/foo'}})
+
+        self.init(testcase, extra_args=['--native-file', config], override_envvars={'PKG_CONFIG_PATH': '/bar'})
+        configuration = self.introspect('--buildoptions')
+        for each in configuration:
+            if each['name'] == 'pkg_config_path':
+                self.assertEqual(each['value'], ['/bar'])
+                break
+        else:
+            self.fail('Did not find pkg_config_path in build options?')
+
     def test_builtin_options_subprojects(self):
         testcase = os.path.join(self.common_test_dir, '102 subproject subdir')
         config = self.helper_create_native_file({'built-in options': {'default_library': 'both', 'c_args': ['-Dfoo']}, 'sub:built-in options': {'default_library': 'static'}})
@@ -8185,6 +8198,22 @@ class NativeFileTests(BasePlatformTests):
         else:
             self.fail('Did not find bindir in build options?')
 
+    def test_builtin_options_paths_legacy(self):
+        testcase = os.path.join(self.common_test_dir, '1 trivial')
+        config = self.helper_create_native_file({
+            'built-in options': {'default_library': 'static'},
+            'paths': {'bindir': 'bar'},
+        })
+
+        self.init(testcase, extra_args=['--native-file', config])
+        configuration = self.introspect('--buildoptions')
+        for each in configuration:
+            if each['name'] == 'bindir':
+                self.assertEqual(each['value'], 'bar')
+                break
+        else:
+            self.fail('Did not find bindir in build options?')
+
 
 class CrossFileTests(BasePlatformTests):
 
@@ -8431,7 +8460,15 @@ class CrossFileTests(BasePlatformTests):
         cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/cross/path', 'cpp_std': 'c++17'}})
         native = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/native/path', 'cpp_std': 'c++14'}})
 
-        self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native])
+        # Ensure that PKG_CONFIG_PATH is not set in the environment
+        with mock.patch.dict('os.environ'):
+            for k in ['PKG_CONFIG_PATH', 'PKG_CONFIG_PATH_FOR_BUILD']:
+                try:
+                    del os.environ[k]
+                except KeyError:
+                    pass
+            self.init(testcase, extra_args=['--cross-file', cross, '--native-file', native])
+
         configuration = self.introspect('--buildoptions')
         found = 0
         for each in configuration:
@@ -8452,6 +8489,26 @@ class CrossFileTests(BasePlatformTests):
                 break
         self.assertEqual(found, 4, 'Did not find all sections.')
 
+    def test_builtin_options_env_overrides_conf(self):
+        testcase = os.path.join(self.common_test_dir, '2 cpp')
+        config = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}})
+        cross = self.helper_create_cross_file({'built-in options': {'pkg_config_path': '/foo'}})
+
+        self.init(testcase, extra_args=['--native-file', config, '--cross-file', cross],
+                  override_envvars={'PKG_CONFIG_PATH': '/bar', 'PKG_CONFIG_PATH_FOR_BUILD': '/dir'})
+        configuration = self.introspect('--buildoptions')
+        found = 0
+        for each in configuration:
+            if each['name'] == 'pkg_config_path':
+                self.assertEqual(each['value'], ['/bar'])
+                found += 1
+            elif each['name'] == 'build.pkg_config_path':
+                self.assertEqual(each['value'], ['/dir'])
+                found += 1
+            if found == 2:
+                break
+        self.assertEqual(found, 2, 'Did not find all sections.')
+
 
 class TAPParserTests(unittest.TestCase):
     def assert_test(self, events, **kwargs):