diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index 76da0b67f..782e77081 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -6,7 +6,7 @@ from __future__ import annotations import copy -from . import mlog, mparser, options +from . import mlog, options import pickle, os, uuid import sys from itertools import chain @@ -16,14 +16,16 @@ from dataclasses import dataclass from .mesonlib import ( MesonBugException, - MesonException, EnvironmentException, MachineChoice, PerMachine, + MesonException, MachineChoice, PerMachine, PerMachineDefaultable, OptionKey, OptionType, stringlistify, pickle_load ) + +from .machinefile import CmdLineFileParser + import ast import argparse -import configparser import enum import shlex import typing as T @@ -760,99 +762,6 @@ class CoreData: mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as \'b_asneeded\' have been disabled.', fatal=False) mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.', fatal=False) -class CmdLineFileParser(configparser.ConfigParser): - def __init__(self) -> None: - # We don't want ':' as key delimiter, otherwise it would break when - # storing subproject options like "subproject:option=value" - super().__init__(delimiters=['='], interpolation=None) - - def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]: - return super().read(filenames, encoding) - - def optionxform(self, optionstr: str) -> str: - # Don't call str.lower() on keys - return optionstr - -class MachineFileParser(): - def __init__(self, filenames: T.List[str], sourcedir: str) -> None: - self.parser = CmdLineFileParser() - self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False} - self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {} - - for fname in filenames: - try: - with open(fname, encoding='utf-8') as f: - content = f.read() - except UnicodeDecodeError as e: - raise EnvironmentException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}') - - content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir) - content = content.replace('@DIRNAME@', os.path.dirname(fname)) - try: - self.parser.read_string(content, fname) - except configparser.Error as e: - raise EnvironmentException(f'Malformed machine file: {e}') - - # Parse [constants] first so they can be used in other sections - if self.parser.has_section('constants'): - self.constants.update(self._parse_section('constants')) - - for s in self.parser.sections(): - if s == 'constants': - continue - self.sections[s] = self._parse_section(s) - - def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]: - self.scope = self.constants.copy() - section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {} - for entry, value in self.parser.items(s): - if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry: - raise EnvironmentException(f'Malformed variable name {entry!r} in machine file.') - # Windows paths... - value = value.replace('\\', '\\\\') - try: - ast = mparser.Parser(value, 'machinefile').parse() - if not ast.lines: - raise EnvironmentException('value cannot be empty') - res = self._evaluate_statement(ast.lines[0]) - except MesonException as e: - raise EnvironmentException(f'Malformed value in machine file variable {entry!r}: {str(e)}.') - except KeyError as e: - raise EnvironmentException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.') - section[entry] = res - self.scope[entry] = res - return section - - def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: - if isinstance(node, (mparser.StringNode)): - return node.value - elif isinstance(node, mparser.BooleanNode): - return node.value - elif isinstance(node, mparser.NumberNode): - return node.value - elif isinstance(node, mparser.ParenthesizedNode): - return self._evaluate_statement(node.inner) - elif isinstance(node, mparser.ArrayNode): - # TODO: This is where recursive types would come in handy - return [self._evaluate_statement(arg) for arg in node.args.arguments] - elif isinstance(node, mparser.IdNode): - return self.scope[node.value] - elif isinstance(node, mparser.ArithmeticNode): - l = self._evaluate_statement(node.left) - r = self._evaluate_statement(node.right) - if node.operation == 'add': - if (isinstance(l, str) and isinstance(r, str)) or \ - (isinstance(l, list) and isinstance(r, list)): - return l + r - elif node.operation == 'div': - if isinstance(l, str) and isinstance(r, str): - return os.path.join(l, r) - raise EnvironmentException('Unsupported node type') - -def parse_machine_files(filenames: T.List[str], sourcedir: str): - parser = MachineFileParser(filenames, sourcedir) - return parser.sections - def get_cmd_line_file(build_dir: str) -> str: return os.path.join(build_dir, 'meson-private', 'cmd_line.txt') diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py index 19b9e81b5..1607c32ed 100644 --- a/mesonbuild/environment.py +++ b/mesonbuild/environment.py @@ -11,6 +11,10 @@ import collections from . import coredata from . import mesonlib +from . import machinefile + +CmdLineFileParser = machinefile.CmdLineFileParser + from .mesonlib import ( MesonException, MachineChoice, Popen_safe, PerMachine, PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey, @@ -589,7 +593,7 @@ class Environment: ## Read in native file(s) to override build machine configuration if self.coredata.config_files is not None: - config = coredata.parse_machine_files(self.coredata.config_files, self.source_dir) + config = machinefile.parse_machine_files(self.coredata.config_files, self.source_dir) binaries.build = BinaryTable(config.get('binaries', {})) properties.build = Properties(config.get('properties', {})) cmakevars.build = CMakeVariables(config.get('cmake', {})) @@ -600,7 +604,7 @@ class Environment: ## Read in cross file(s) to override host machine configuration if self.coredata.cross_files: - config = coredata.parse_machine_files(self.coredata.cross_files, self.source_dir) + config = machinefile.parse_machine_files(self.coredata.cross_files, self.source_dir) properties.host = Properties(config.get('properties', {})) binaries.host = BinaryTable(config.get('binaries', {})) cmakevars.host = CMakeVariables(config.get('cmake', {})) diff --git a/mesonbuild/machinefile.py b/mesonbuild/machinefile.py index 18162eff5..afeb4d056 100644 --- a/mesonbuild/machinefile.py +++ b/mesonbuild/machinefile.py @@ -1,15 +1,117 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright 2013-2024 Contributors to the The Meson project -from . import mlog +import typing as T +import configparser +import os + +from . import mparser + +from .mesonlib import MesonException + +if T.TYPE_CHECKING: + from .compilers import Compiler + from .coredata import StrOrBytesPath + + CompilersDict = T.Dict[str, Compiler] + + +class CmdLineFileParser(configparser.ConfigParser): + def __init__(self) -> None: + # We don't want ':' as key delimiter, otherwise it would break when + # storing subproject options like "subproject:option=value" + super().__init__(delimiters=['='], interpolation=None) + + def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]: + return super().read(filenames, encoding) + + def optionxform(self, optionstr: str) -> str: + # Don't call str.lower() on keys + return optionstr + + +class MachineFileParser(): + def __init__(self, filenames: T.List[str], sourcedir: str) -> None: + self.parser = CmdLineFileParser() + self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False} + self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {} + + for fname in filenames: + try: + with open(fname, encoding='utf-8') as f: + content = f.read() + except UnicodeDecodeError as e: + raise MesonException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}') + + content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir) + content = content.replace('@DIRNAME@', os.path.dirname(fname)) + try: + self.parser.read_string(content, fname) + except configparser.Error as e: + raise MesonException(f'Malformed machine file: {e}') + + # Parse [constants] first so they can be used in other sections + if self.parser.has_section('constants'): + self.constants.update(self._parse_section('constants')) + + for s in self.parser.sections(): + if s == 'constants': + continue + self.sections[s] = self._parse_section(s) + + def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]: + self.scope = self.constants.copy() + section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {} + for entry, value in self.parser.items(s): + if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry: + raise MesonException(f'Malformed variable name {entry!r} in machine file.') + # Windows paths... + value = value.replace('\\', '\\\\') + try: + ast = mparser.Parser(value, 'machinefile').parse() + if not ast.lines: + raise MesonException('value cannot be empty') + res = self._evaluate_statement(ast.lines[0]) + except MesonException as e: + raise MesonException(f'Malformed value in machine file variable {entry!r}: {str(e)}.') + except KeyError as e: + raise MesonException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.') + section[entry] = res + self.scope[entry] = res + return section + + def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: + if isinstance(node, (mparser.StringNode)): + return node.value + elif isinstance(node, mparser.BooleanNode): + return node.value + elif isinstance(node, mparser.NumberNode): + return node.value + elif isinstance(node, mparser.ParenthesizedNode): + return self._evaluate_statement(node.inner) + elif isinstance(node, mparser.ArrayNode): + # TODO: This is where recursive types would come in handy + return [self._evaluate_statement(arg) for arg in node.args.arguments] + elif isinstance(node, mparser.IdNode): + return self.scope[node.value] + elif isinstance(node, mparser.ArithmeticNode): + l = self._evaluate_statement(node.left) + r = self._evaluate_statement(node.right) + if node.operation == 'add': + if (isinstance(l, str) and isinstance(r, str)) or \ + (isinstance(l, list) and isinstance(r, list)): + return l + r + elif node.operation == 'div': + if isinstance(l, str) and isinstance(r, str): + return os.path.join(l, r) + raise MesonException('Unsupported node type') + +def parse_machine_files(filenames: T.List[str], sourcedir: str): + parser = MachineFileParser(filenames, sourcedir) + return parser.sections -class MachineFile: - def __init__(self, fname): - with open(fname, encoding='utf-8') as f: - pass - self.stuff = None class MachineFileStore: - def __init__(self, native_files, cross_files): - self.native = [MachineFile(x) for x in native_files] - self.cross = [MachineFile(x) for x in cross_files] + def __init__(self, native_files, cross_files, source_dir): + self.native = MachineFileParser(native_files if native_files is not None else [], source_dir).sections + self.cross = MachineFileParser(cross_files if cross_files is not None else [], source_dir).sections diff --git a/mesonbuild/scripts/scanbuild.py b/mesonbuild/scripts/scanbuild.py index d7fbcf4fe..b738aeee1 100644 --- a/mesonbuild/scripts/scanbuild.py +++ b/mesonbuild/scripts/scanbuild.py @@ -7,7 +7,8 @@ import subprocess import shutil import tempfile from ..environment import detect_ninja, detect_scanbuild -from ..coredata import get_cmd_line_file, CmdLineFileParser +from ..coredata import get_cmd_line_file +from ..machinefile import CmdLineFileParser from ..mesonlib import windows_proof_rmtree from pathlib import Path import typing as T diff --git a/test cases/unit/116 empty project/expected_mods.json b/test cases/unit/116 empty project/expected_mods.json index 19f56a5f7..fa5e0ec6c 100644 --- a/test cases/unit/116 empty project/expected_mods.json +++ b/test cases/unit/116 empty project/expected_mods.json @@ -217,6 +217,7 @@ "mesonbuild.linkers", "mesonbuild.linkers.base", "mesonbuild.linkers.detect", + "mesonbuild.machinefile", "mesonbuild.mesonlib", "mesonbuild.mesonmain", "mesonbuild.mintro", diff --git a/unittests/machinefiletests.py b/unittests/machinefiletests.py index 3899ea942..5ff862cdc 100644 --- a/unittests/machinefiletests.py +++ b/unittests/machinefiletests.py @@ -59,7 +59,7 @@ class MachineFileStoreTests(TestCase): def test_loading(self): store = machinefile.MachineFileStore([cross_dir / 'ubuntu-armhf.txt'], [], str(cross_dir)) - self.assertTrue(True) + self.assertIsNotNone(store) class NativeFileTests(BasePlatformTests): diff --git a/unittests/platformagnostictests.py b/unittests/platformagnostictests.py index 33a789b4b..fe598a72b 100644 --- a/unittests/platformagnostictests.py +++ b/unittests/platformagnostictests.py @@ -274,7 +274,7 @@ class PlatformAgnosticTests(BasePlatformTests): expected = json.load(f)['meson']['modules'] self.assertEqual(data['modules'], expected) - self.assertEqual(data['count'], 69) + self.assertEqual(data['count'], 70) def test_meson_package_cache_dir(self): # Copy testdir into temporary directory to not pollute meson source tree.