Extract native file parser to machinefile source file.

opthelper
Jussi Pakkanen 6 months ago
parent 902252b841
commit c14ff1b31e
  1. 101
      mesonbuild/coredata.py
  2. 8
      mesonbuild/environment.py
  3. 120
      mesonbuild/machinefile.py
  4. 3
      mesonbuild/scripts/scanbuild.py
  5. 1
      test cases/unit/116 empty project/expected_mods.json
  6. 2
      unittests/machinefiletests.py
  7. 2
      unittests/platformagnostictests.py

@ -6,7 +6,7 @@ from __future__ import annotations
import copy
from . import mlog, mparser, options
from . import mlog, options
import pickle, os, uuid
import sys
from itertools import chain
@ -16,14 +16,16 @@ from dataclasses import dataclass
from .mesonlib import (
MesonBugException,
MesonException, EnvironmentException, MachineChoice, PerMachine,
MesonException, MachineChoice, PerMachine,
PerMachineDefaultable,
OptionKey, OptionType, stringlistify,
pickle_load
)
from .machinefile import CmdLineFileParser
import ast
import argparse
import configparser
import enum
import shlex
import typing as T
@ -760,99 +762,6 @@ class CoreData:
mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as \'b_asneeded\' have been disabled.', fatal=False)
mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.', fatal=False)
class CmdLineFileParser(configparser.ConfigParser):
def __init__(self) -> None:
# We don't want ':' as key delimiter, otherwise it would break when
# storing subproject options like "subproject:option=value"
super().__init__(delimiters=['='], interpolation=None)
def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
return super().read(filenames, encoding)
def optionxform(self, optionstr: str) -> str:
# Don't call str.lower() on keys
return optionstr
class MachineFileParser():
def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
self.parser = CmdLineFileParser()
self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
for fname in filenames:
try:
with open(fname, encoding='utf-8') as f:
content = f.read()
except UnicodeDecodeError as e:
raise EnvironmentException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
content = content.replace('@DIRNAME@', os.path.dirname(fname))
try:
self.parser.read_string(content, fname)
except configparser.Error as e:
raise EnvironmentException(f'Malformed machine file: {e}')
# Parse [constants] first so they can be used in other sections
if self.parser.has_section('constants'):
self.constants.update(self._parse_section('constants'))
for s in self.parser.sections():
if s == 'constants':
continue
self.sections[s] = self._parse_section(s)
def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
self.scope = self.constants.copy()
section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
for entry, value in self.parser.items(s):
if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
raise EnvironmentException(f'Malformed variable name {entry!r} in machine file.')
# Windows paths...
value = value.replace('\\', '\\\\')
try:
ast = mparser.Parser(value, 'machinefile').parse()
if not ast.lines:
raise EnvironmentException('value cannot be empty')
res = self._evaluate_statement(ast.lines[0])
except MesonException as e:
raise EnvironmentException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
except KeyError as e:
raise EnvironmentException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
section[entry] = res
self.scope[entry] = res
return section
def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
if isinstance(node, (mparser.StringNode)):
return node.value
elif isinstance(node, mparser.BooleanNode):
return node.value
elif isinstance(node, mparser.NumberNode):
return node.value
elif isinstance(node, mparser.ParenthesizedNode):
return self._evaluate_statement(node.inner)
elif isinstance(node, mparser.ArrayNode):
# TODO: This is where recursive types would come in handy
return [self._evaluate_statement(arg) for arg in node.args.arguments]
elif isinstance(node, mparser.IdNode):
return self.scope[node.value]
elif isinstance(node, mparser.ArithmeticNode):
l = self._evaluate_statement(node.left)
r = self._evaluate_statement(node.right)
if node.operation == 'add':
if (isinstance(l, str) and isinstance(r, str)) or \
(isinstance(l, list) and isinstance(r, list)):
return l + r
elif node.operation == 'div':
if isinstance(l, str) and isinstance(r, str):
return os.path.join(l, r)
raise EnvironmentException('Unsupported node type')
def parse_machine_files(filenames: T.List[str], sourcedir: str):
parser = MachineFileParser(filenames, sourcedir)
return parser.sections
def get_cmd_line_file(build_dir: str) -> str:
return os.path.join(build_dir, 'meson-private', 'cmd_line.txt')

@ -11,6 +11,10 @@ import collections
from . import coredata
from . import mesonlib
from . import machinefile
CmdLineFileParser = machinefile.CmdLineFileParser
from .mesonlib import (
MesonException, MachineChoice, Popen_safe, PerMachine,
PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey,
@ -589,7 +593,7 @@ class Environment:
## Read in native file(s) to override build machine configuration
if self.coredata.config_files is not None:
config = coredata.parse_machine_files(self.coredata.config_files, self.source_dir)
config = machinefile.parse_machine_files(self.coredata.config_files, self.source_dir)
binaries.build = BinaryTable(config.get('binaries', {}))
properties.build = Properties(config.get('properties', {}))
cmakevars.build = CMakeVariables(config.get('cmake', {}))
@ -600,7 +604,7 @@ class Environment:
## Read in cross file(s) to override host machine configuration
if self.coredata.cross_files:
config = coredata.parse_machine_files(self.coredata.cross_files, self.source_dir)
config = machinefile.parse_machine_files(self.coredata.cross_files, self.source_dir)
properties.host = Properties(config.get('properties', {}))
binaries.host = BinaryTable(config.get('binaries', {}))
cmakevars.host = CMakeVariables(config.get('cmake', {}))

@ -1,15 +1,117 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2013-2024 Contributors to the The Meson project
from . import mlog
import typing as T
import configparser
import os
from . import mparser
from .mesonlib import MesonException
if T.TYPE_CHECKING:
from .compilers import Compiler
from .coredata import StrOrBytesPath
CompilersDict = T.Dict[str, Compiler]
class CmdLineFileParser(configparser.ConfigParser):
def __init__(self) -> None:
# We don't want ':' as key delimiter, otherwise it would break when
# storing subproject options like "subproject:option=value"
super().__init__(delimiters=['='], interpolation=None)
def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
return super().read(filenames, encoding)
def optionxform(self, optionstr: str) -> str:
# Don't call str.lower() on keys
return optionstr
class MachineFileParser():
def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
self.parser = CmdLineFileParser()
self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
for fname in filenames:
try:
with open(fname, encoding='utf-8') as f:
content = f.read()
except UnicodeDecodeError as e:
raise MesonException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
content = content.replace('@DIRNAME@', os.path.dirname(fname))
try:
self.parser.read_string(content, fname)
except configparser.Error as e:
raise MesonException(f'Malformed machine file: {e}')
# Parse [constants] first so they can be used in other sections
if self.parser.has_section('constants'):
self.constants.update(self._parse_section('constants'))
for s in self.parser.sections():
if s == 'constants':
continue
self.sections[s] = self._parse_section(s)
def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
self.scope = self.constants.copy()
section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
for entry, value in self.parser.items(s):
if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
raise MesonException(f'Malformed variable name {entry!r} in machine file.')
# Windows paths...
value = value.replace('\\', '\\\\')
try:
ast = mparser.Parser(value, 'machinefile').parse()
if not ast.lines:
raise MesonException('value cannot be empty')
res = self._evaluate_statement(ast.lines[0])
except MesonException as e:
raise MesonException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
except KeyError as e:
raise MesonException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
section[entry] = res
self.scope[entry] = res
return section
def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
if isinstance(node, (mparser.StringNode)):
return node.value
elif isinstance(node, mparser.BooleanNode):
return node.value
elif isinstance(node, mparser.NumberNode):
return node.value
elif isinstance(node, mparser.ParenthesizedNode):
return self._evaluate_statement(node.inner)
elif isinstance(node, mparser.ArrayNode):
# TODO: This is where recursive types would come in handy
return [self._evaluate_statement(arg) for arg in node.args.arguments]
elif isinstance(node, mparser.IdNode):
return self.scope[node.value]
elif isinstance(node, mparser.ArithmeticNode):
l = self._evaluate_statement(node.left)
r = self._evaluate_statement(node.right)
if node.operation == 'add':
if (isinstance(l, str) and isinstance(r, str)) or \
(isinstance(l, list) and isinstance(r, list)):
return l + r
elif node.operation == 'div':
if isinstance(l, str) and isinstance(r, str):
return os.path.join(l, r)
raise MesonException('Unsupported node type')
def parse_machine_files(filenames: T.List[str], sourcedir: str):
parser = MachineFileParser(filenames, sourcedir)
return parser.sections
class MachineFile:
def __init__(self, fname):
with open(fname, encoding='utf-8') as f:
pass
self.stuff = None
class MachineFileStore:
def __init__(self, native_files, cross_files):
self.native = [MachineFile(x) for x in native_files]
self.cross = [MachineFile(x) for x in cross_files]
def __init__(self, native_files, cross_files, source_dir):
self.native = MachineFileParser(native_files if native_files is not None else [], source_dir).sections
self.cross = MachineFileParser(cross_files if cross_files is not None else [], source_dir).sections

@ -7,7 +7,8 @@ import subprocess
import shutil
import tempfile
from ..environment import detect_ninja, detect_scanbuild
from ..coredata import get_cmd_line_file, CmdLineFileParser
from ..coredata import get_cmd_line_file
from ..machinefile import CmdLineFileParser
from ..mesonlib import windows_proof_rmtree
from pathlib import Path
import typing as T

@ -217,6 +217,7 @@
"mesonbuild.linkers",
"mesonbuild.linkers.base",
"mesonbuild.linkers.detect",
"mesonbuild.machinefile",
"mesonbuild.mesonlib",
"mesonbuild.mesonmain",
"mesonbuild.mintro",

@ -59,7 +59,7 @@ class MachineFileStoreTests(TestCase):
def test_loading(self):
store = machinefile.MachineFileStore([cross_dir / 'ubuntu-armhf.txt'], [], str(cross_dir))
self.assertTrue(True)
self.assertIsNotNone(store)
class NativeFileTests(BasePlatformTests):

@ -274,7 +274,7 @@ class PlatformAgnosticTests(BasePlatformTests):
expected = json.load(f)['meson']['modules']
self.assertEqual(data['modules'], expected)
self.assertEqual(data['count'], 69)
self.assertEqual(data['count'], 70)
def test_meson_package_cache_dir(self):
# Copy testdir into temporary directory to not pollute meson source tree.

Loading…
Cancel
Save