From b25a423a645491e83112929f95c1bd9312458a9a Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Mon, 30 Nov 2020 12:10:40 -0800 Subject: [PATCH] use the OptionKey type for command line and machine files --- mesonbuild/ast/introspection.py | 6 +- mesonbuild/coredata.py | 125 +++++++++++++------------------- mesonbuild/environment.py | 93 +++++++++++++----------- mesonbuild/interpreter.py | 7 +- mesonbuild/mconf.py | 2 +- run_unittests.py | 2 +- 6 files changed, 111 insertions(+), 124 deletions(-) diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index eca869fa3..086174805 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -65,7 +65,7 @@ class IntrospectionInterpreter(AstInterpreter): self.coredata = self.environment.get_coredata() self.option_file = os.path.join(self.source_root, self.subdir, 'meson_options.txt') self.backend = backend - self.default_options = {'backend': self.backend} + self.default_options = {cdata.OptionKey('backend'): self.backend} self.project_data = {} # type: T.Dict[str, T.Any] self.targets = [] # type: T.List[T.Dict[str, T.Any]] self.dependencies = [] # type: T.List[T.Dict[str, T.Any]] @@ -107,7 +107,7 @@ class IntrospectionInterpreter(AstInterpreter): def_opts = self.flatten_args(kwargs.get('default_options', [])) _project_default_options = mesonlib.stringlistify(def_opts) - self.project_default_options = cdata.create_options_dict(_project_default_options) + self.project_default_options = cdata.create_options_dict(_project_default_options, self.subproject) self.default_options.update(self.project_default_options) self.coredata.set_default_options(self.default_options, self.subproject, self.environment) @@ -125,7 +125,7 @@ class IntrospectionInterpreter(AstInterpreter): self.do_subproject(i) self.coredata.init_backend_options(self.backend) - options = {k: v for k, v in self.environment.raw_options.items() if k.startswith('backend_')} + options = {k: v for k, v in self.environment.options.items() if k.name.startswith('backend_')} self.coredata.set_options(options) self._add_languages(proj_langs, MachineChoice.HOST) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index 8b272e853..b39f205d7 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -551,7 +551,7 @@ class CoreData: self.target_guids = {} self.version = version self.builtins = {} # type: OptionDictType - self.builtins_per_machine = PerMachine({}, {}) + self.builtins_per_machine: PerMachine['OptionDictType'] = PerMachine({}, {}) self.backend_options = {} # type: OptionDictType self.user_options = {} # type: OptionDictType self.compiler_options = PerMachine( @@ -835,14 +835,10 @@ class CoreData: return k[:idx + 1] + 'build.' + k[idx + 1:] @classmethod - def is_per_machine_option(cls, optname): - if optname in BUILTIN_OPTIONS_PER_MACHINE: + def is_per_machine_option(cls, optname: OptionKey) -> bool: + if optname.name in BUILTIN_OPTIONS_PER_MACHINE: return True - from .compilers import compilers - for lang_prefix in [lang + '_' for lang in compilers.all_languages]: - if optname.startswith(lang_prefix): - return True - return False + return optname.lang is not None def _get_all_nonbuiltin_options(self) -> T.Iterable[T.Dict[str, UserOption]]: yield self.backend_options @@ -899,16 +895,6 @@ class CoreData: return False return len(self.cross_files) > 0 - def strip_build_option_names(self, options): - res = OrderedDict() - for k, v in options.items(): - if k.startswith('build.'): - k = k.split('.', 1)[1] - res.setdefault(k, v) - else: - res[k] = v - return res - def copy_build_options_from_regular_ones(self): assert(not self.is_cross_build()) for k, o in self.builtins_per_machine.host.items(): @@ -919,25 +905,26 @@ class CoreData: if k in build_opts: build_opts[k].set_value(o.value) - def set_options(self, options: T.Dict[str, T.Any], subproject: str = '', warn_unknown: bool = True) -> None: + def set_options(self, options: T.Dict[OptionKey, T.Any], subproject: str = '', warn_unknown: bool = True) -> None: if not self.is_cross_build(): - options = self.strip_build_option_names(options) + options = {k: v for k, v in options.items() if k.machine is not MachineChoice.BUILD} # Set prefix first because it's needed to sanitize other options - if 'prefix' in options: - prefix = self.sanitize_prefix(options['prefix']) + pfk = OptionKey('prefix') + if pfk in options: + prefix = self.sanitize_prefix(options[pfk]) self.builtins['prefix'].set_value(prefix) for key in builtin_dir_noprefix_options: if key not in options: self.builtins[key].set_value(BUILTIN_OPTIONS[key].prefixed_default(key, prefix)) - unknown_options = [] + unknown_options: T.List[OptionKey] = [] for k, v in options.items(): - if k == 'prefix': + if k == pfk: continue - if self._try_set_builtin_option(k, v): + if self._try_set_builtin_option(str(k), v): continue for opts in self._get_all_nonbuiltin_options(): - tgt = opts.get(k) + tgt = opts.get(str(k)) if tgt is None: continue tgt.set_value(v) @@ -945,59 +932,50 @@ class CoreData: else: unknown_options.append(k) if unknown_options and warn_unknown: - unknown_options = ', '.join(sorted(unknown_options)) + unknown_options_str = ', '.join(sorted(str(s) for s in unknown_options)) sub = 'In subproject {}: '.format(subproject) if subproject else '' - mlog.warning('{}Unknown options: "{}"'.format(sub, unknown_options)) + mlog.warning('{}Unknown options: "{}"'.format(sub, unknown_options_str)) mlog.log('The value of new options can be set with:') mlog.log(mlog.bold('meson setup --reconfigure -Dnew_option=new_value ...')) if not self.is_cross_build(): self.copy_build_options_from_regular_ones() - def set_default_options(self, default_options: 'T.OrderedDict[str, str]', subproject: str, env: 'Environment') -> None: - # Preserve order: if env.raw_options has 'buildtype' it must come after + def set_default_options(self, default_options: T.MutableMapping[OptionKey, str], subproject: str, env: 'Environment') -> None: + # Preserve order: if env.options has 'buildtype' it must come after # 'optimization' if it is in default_options. - raw_options = OrderedDict() - for k, v in default_options.items(): - if subproject: - k = subproject + ':' + k - raw_options[k] = v - raw_options.update(env.raw_options) - env.raw_options = raw_options - - # Create a subset of raw_options, keeping only project and builtin + if not subproject: + options: T.MutableMapping[OptionKey, T.Any] = OrderedDict() + for k, v in default_options.items(): + options[k] = v + options.update(env.options) + env.options = options + + # Create a subset of options, keeping only project and builtin # options for this subproject. # Language and backend specific options will be set later when adding # languages and setting the backend (builtin options must be set first # to know which backend we'll use). - options = OrderedDict() + options: T.MutableMapping[OptionKey, T.Any] = OrderedDict() from . import optinterpreter - for k, v in env.raw_options.items(): - raw_optname = k - if subproject: - # Subproject: skip options for other subprojects - if not k.startswith(subproject + ':'): - continue - raw_optname = k.split(':')[1] - elif ':' in k: - # Main prject: skip options for subprojects + for k, v in chain(default_options.items(), env.options.items()): + # Subproject: skip options for other subprojects + if k.subproject and k.subproject != subproject: continue # Skip base, compiler, and backend options, they are handled when # adding languages and setting backend. - if (k not in self.builtins and - k not in self.get_prefixed_options_per_machine(self.builtins_per_machine) and - optinterpreter.is_invalid_name(raw_optname, log=False)): + if (k.name not in self.builtins and k.name not in self.builtins_per_machine[k.machine] and + optinterpreter.is_invalid_name(str(k), log=False)): continue options[k] = v self.set_options(options, subproject=subproject) - def add_compiler_options(self, options, lang, for_machine, env): - # prefixed compiler options affect just this machine - opt_prefix = for_machine.get_prefix() + def add_compiler_options(self, options: 'OptionDictType', lang: str, for_machine: MachineChoice, + env: 'Environment') -> None: for k, o in options.items(): - optname = opt_prefix + lang + '_' + k - value = env.raw_options.get(optname) + key = OptionKey(k, lang=lang, machine=for_machine) + value = env.options.get(key) if value is not None: o.set_value(value) self.compiler_options[for_machine][lang].setdefault(k, o) @@ -1006,8 +984,7 @@ class CoreData: for_machine: MachineChoice, env: 'Environment') -> None: """Add global language arguments that are needed before compiler/linker detection.""" from .compilers import compilers - options = compilers.get_global_options(lang, comp, for_machine, - env.is_cross_build()) + options = compilers.get_global_options(lang, comp, for_machine, env.is_cross_build()) self.add_compiler_options(options, lang, for_machine, env) def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None: @@ -1021,8 +998,9 @@ class CoreData: if optname in self.base_options: continue oobj = compilers.base_options[optname] - if optname in env.raw_options: - oobj.set_value(env.raw_options[optname]) + key = OptionKey(optname, machine=comp.for_machine) + if key in env.options: + oobj.set_value(env.options[key]) enabled_opts.append(optname) self.base_options[optname] = oobj self.emit_base_options_warnings(enabled_opts) @@ -1118,7 +1096,7 @@ def read_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None: # Do a copy because config is not really a dict. options.cmd_line_options # overrides values from the file. - d = dict(config['options']) + d = {OptionKey.from_string(k): v for k, v in config['options'].items()} d.update(options.cmd_line_options) options.cmd_line_options = d @@ -1130,9 +1108,6 @@ def read_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None: # literal_eval to get it into the list of strings. options.native_file = ast.literal_eval(properties.get('native_file', '[]')) -def cmd_line_options_to_string(options: argparse.Namespace) -> T.Dict[str, str]: - return {k: str(v) for k, v in options.cmd_line_options.items()} - def write_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None: filename = get_cmd_line_file(build_dir) config = CmdLineFileParser() @@ -1143,7 +1118,7 @@ def write_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None: if options.native_file: properties['native_file'] = [os.path.abspath(f) for f in options.native_file] - config['options'] = cmd_line_options_to_string(options) + config['options'] = {str(k): str(v) for k, v in options.cmd_line_options.items()} config['properties'] = properties with open(filename, 'w') as f: config.write(f) @@ -1152,14 +1127,14 @@ def update_cmd_line_file(build_dir: str, options: argparse.Namespace): filename = get_cmd_line_file(build_dir) config = CmdLineFileParser() config.read(filename) - config['options'].update(cmd_line_options_to_string(options)) + config['options'].update({str(k): str(v) for k, v in options.cmd_line_options.items()}) with open(filename, 'w') as f: config.write(f) def get_cmd_line_options(build_dir: str, options: argparse.Namespace) -> str: copy = argparse.Namespace(**vars(options)) read_cmd_line_file(build_dir, copy) - cmdline = ['-D{}={}'.format(k, v) for k, v in copy.cmd_line_options.items()] + cmdline = ['-D{}={}'.format(str(k), v) for k, v in copy.cmd_line_options.items()] if options.cross_file: cmdline += ['--cross-file {}'.format(f) for f in options.cross_file] if options.native_file: @@ -1214,14 +1189,17 @@ def register_builtin_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument('-D', action='append', dest='projectoptions', default=[], metavar="option", help='Set the value of an option, can be used several times to set multiple options.') -def create_options_dict(options: T.List[str]) -> T.Dict[str, str]: - result = OrderedDict() +def create_options_dict(options: T.List[str], subproject: str = '') -> T.Dict[OptionKey, str]: + result: T.OrderedDict[OptionKey, str] = OrderedDict() for o in options: try: (key, value) = o.split('=', 1) except ValueError: raise MesonException('Option {!r} must have a value separated by equals sign.'.format(o)) - result[key] = value + k = OptionKey.from_string(key) + if subproject: + k = k.evolve(subproject=subproject) + result[k] = value return result def parse_cmd_line_options(args: argparse.Namespace) -> None: @@ -1235,11 +1213,12 @@ def parse_cmd_line_options(args: argparse.Namespace) -> None: ): value = getattr(args, name, None) if value is not None: - if name in args.cmd_line_options: + key = OptionKey.from_string(name) + if key in args.cmd_line_options: cmdline_name = BuiltinOption.argparse_name_to_arg(name) raise MesonException( 'Got argument {0} as both -D{0} and {1}. Pick one.'.format(name, cmdline_name)) - args.cmd_line_options[name] = value + args.cmd_line_options[key] = value delattr(args, name) diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 74d8bde1c..d2a118fde 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -130,6 +130,8 @@ from .compilers import ( ) if T.TYPE_CHECKING: + from configparser import ConfigParser + from .dependencies import ExternalProgram build_filename = 'meson.build' @@ -599,12 +601,11 @@ class Environment: binaries.build = BinaryTable() properties.build = Properties() - # Unparsed options as given by the user in machine files, command line, - # and project()'s default_options. Keys are in the command line format: - # "[:][build.]option_name". + # Options with the key parsed into an OptionKey type. + # # Note that order matters because of 'buildtype', if it is after # 'optimization' and 'debug' keys, it override them. - self.raw_options = collections.OrderedDict() # type: collections.OrderedDict[str, str] + self.options: T.MutableMapping[coredata.OptionKey, str] = collections.OrderedDict() ## Read in native file(s) to override build machine configuration @@ -613,7 +614,7 @@ class Environment: binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) cmakevars.build = CMakeVariables(config.get('cmake', {})) - self.load_machine_file_options(config, properties.build) + self.load_machine_file_options(config, properties.build, MachineChoice.BUILD) ## Read in cross file(s) to override host machine configuration @@ -626,10 +627,16 @@ class Environment: machines.host = MachineInfo.from_literal(config['host_machine']) if 'target_machine' in config: machines.target = MachineInfo.from_literal(config['target_machine']) - # Keep only per machine options from the native file and prefix them - # with "build.". The cross file takes precedence over all other options. + # Keep only per machine options from the native file. The cross + # file takes precedence over all other options. self.keep_per_machine_options() - self.load_machine_file_options(config, properties.host) + self.load_machine_file_options(config, properties.host, MachineChoice.HOST) + else: + # IF we aren't cross compiling, but we hav ea native file, the + # native file is for the host. This is due to an mismatch between + # meson internals which talk about build an host, and external + # interfaces which talk about native and cross. + self.options = {k.as_host(): v for k, v in self.options.items()} ## "freeze" now initialized configuration, and "save" to the class. @@ -639,15 +646,17 @@ class Environment: self.cmakevars = cmakevars.default_missing() # Command line options override those from cross/native files - self.raw_options.update(options.cmd_line_options) + self.options.update(options.cmd_line_options) # Take default value from env if not set in cross/native files or command line. self.set_default_options_from_env() # Warn if the user is using two different ways of setting build-type # options that override each other - if 'buildtype' in self.raw_options and \ - ('optimization' in self.raw_options or 'debug' in self.raw_options): + bt = coredata.OptionKey('buildtype') + db = coredata.OptionKey('debug') + op = coredata.OptionKey('optimization') + if bt in self.options and (db in self.options or op in self.options): mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. ' 'Using both is redundant since they override each other. ' 'See: https://mesonbuild.com/Builtin-options.html#build-type-options') @@ -706,11 +715,13 @@ class Environment: self.default_pkgconfig = ['pkg-config'] self.wrap_resolver = None - def load_machine_file_options(self, config, properties): + def load_machine_file_options(self, config: 'ConfigParser', properties: Properties, machine: MachineChoice) -> None: + """Read the contents of a Machine file and put it in the options store.""" paths = config.get('paths') if paths: mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.') - self.raw_options.update(paths) + for k, v in paths.items(): + self.options[coredata.OptionKey.from_string(k).evolve(machine=machine)] = v deprecated_properties = set() for lang in compilers.all_languages: deprecated_properties.add(lang + '_args') @@ -718,44 +729,40 @@ class Environment: for k, v in properties.properties.copy().items(): if k in deprecated_properties: mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k)) - self.raw_options[k] = v + self.options[coredata.OptionKey.from_string(k).evolve(machine=machine)] = v del properties.properties[k] for section, values in config.items(): - prefix = '' if ':' in section: subproject, section = section.split(':') - prefix = subproject + ':' + else: + subproject = '' if section in ['project options', 'built-in options']: - self.raw_options.update({prefix + k: v for k, v in values.items()}) + for k, v in values.items(): + key = coredata.OptionKey.from_string(k).evolve(subproject=subproject, machine=machine) + self.options[key] = v - def keep_per_machine_options(self): - per_machine_options = {} - for optname, value in self.raw_options.items(): - if self.coredata.is_per_machine_option(optname): - build_optname = self.coredata.insert_build_prefix(optname) - per_machine_options[build_optname] = value - self.raw_options = per_machine_options + def keep_per_machine_options(self) -> None: + for key, value in self.options.items(): + if self.coredata.is_per_machine_option(key): + self.options[key.as_build()] = value - def set_default_options_from_env(self): + def set_default_options_from_env(self) -> None: for for_machine in MachineChoice: - p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), 'PKG_CONFIG_PATH') - if p_env_pair is not None: - p_env_var, p_env = p_env_pair - - # PKG_CONFIG_PATH may contain duplicates, which must be - # removed, else a duplicates-in-array-option warning arises. - p_list = list(mesonlib.OrderedSet(p_env.split(':'))) - - key = 'pkg_config_path' - if for_machine == MachineChoice.BUILD: - key = 'build.' + key - - # Take env vars only on first invocation, if the env changes when - # reconfiguring it gets ignored. - # FIXME: We should remember if we took the value from env to warn - # if it changes on future invocations. - if self.first_invocation: - self.raw_options.setdefault(key, p_list) + for evar, keyname in [('PKG_CONFIG_PATH', 'pkg_config_path')]: + p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), evar) + if p_env_pair is not None: + _, p_env = p_env_pair + + # PKG_CONFIG_PATH may contain duplicates, which must be + # removed, else a duplicates-in-array-option warning arises. + p_list = list(mesonlib.OrderedSet(p_env.split(':'))) + # Take env vars only on first invocation, if the env changes when + # reconfiguring it gets ignored. + # FIXME: We should remember if we took the value from env to warn + # if it changes on future invocations. + if self.first_invocation: + key = coredata.OptionKey(keyname, machine=for_machine) + self.options.setdefault(key, p_list) def create_new_coredata(self, options: 'argparse.Namespace') -> None: # WARNING: Don't use any values from coredata in __init__. It gets diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py index a4a9fb2fa..c5cac327f 100644 --- a/mesonbuild/interpreter.py +++ b/mesonbuild/interpreter.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pdb from . import mparser from . import environment from . import coredata @@ -2905,7 +2906,7 @@ external dependencies (including libraries) must go to "dependencies".''') return self.disabled_subproject(subp_name, disabled_feature=feature) default_options = mesonlib.stringlistify(kwargs.get('default_options', [])) - default_options = coredata.create_options_dict(default_options) + default_options = coredata.create_options_dict(default_options, subp_name) if subp_name == '': raise InterpreterException('Subproject name must not be empty.') @@ -3140,7 +3141,7 @@ external dependencies (including libraries) must go to "dependencies".''') if self.environment.first_invocation: self.coredata.init_backend_options(backend) - options = {k: v for k, v in self.environment.raw_options.items() if k.startswith('backend_')} + options = {k: v for k, v in self.environment.options.items() if k.name.startswith('backend_')} self.coredata.set_options(options) @stringArgs @@ -3172,7 +3173,7 @@ external dependencies (including libraries) must go to "dependencies".''') # default_options in a project will trigger a reconfigure but won't # have any effect. self.project_default_options = mesonlib.stringlistify(kwargs.get('default_options', [])) - self.project_default_options = coredata.create_options_dict(self.project_default_options) + self.project_default_options = coredata.create_options_dict(self.project_default_options, self.subproject) if self.environment.first_invocation: default_options = self.project_default_options.copy() default_options.update(self.default_project_options) diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py index 774dc5a07..46aac6ea6 100644 --- a/mesonbuild/mconf.py +++ b/mesonbuild/mconf.py @@ -251,7 +251,7 @@ def run(options): return 0 save = False - if len(options.cmd_line_options) > 0: + if options.cmd_line_options: c.set_options(options.cmd_line_options) coredata.update_cmd_line_file(builddir, options) save = True diff --git a/run_unittests.py b/run_unittests.py index 44a98e53b..83c18541d 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -6904,7 +6904,7 @@ class LinuxlikeTests(BasePlatformTests): self.assertTrue(os.path.exists(os.path.join(pkg_dir, 'librelativepath.pc'))) env = get_fake_env(testdir, self.builddir, self.prefix) - env.coredata.set_options({'pkg_config_path': pkg_dir}, subproject='') + env.coredata.set_options({mesonbuild.coredata.OptionKey('pkg_config_path'): pkg_dir}, subproject='') kwargs = {'required': True, 'silent': True} relative_path_dep = PkgConfigDependency('librelativepath', env, kwargs) self.assertTrue(relative_path_dep.found())