Extract native file parser to machinefile source file.

mesonbuild/coredata.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import copy
 
-from . import mlog, mparser, options
+from . import mlog, options
 import pickle, os, uuid
 import sys
 from itertools import chain
@@ -16,14 +16,16 @@ from dataclasses import dataclass
 
 from .mesonlib import (
     MesonBugException,
-    MesonException, EnvironmentException, MachineChoice, PerMachine,
+    MesonException, MachineChoice, PerMachine,
     PerMachineDefaultable,
     OptionKey, OptionType, stringlistify,
     pickle_load
 )
+
+from .machinefile import CmdLineFileParser
+
 import ast
 import argparse
-import configparser
 import enum
 import shlex
 import typing as T
@@ -760,99 +762,6 @@ class CoreData:
             mlog.warning('Base option \'b_bitcode\' is enabled, which is incompatible with many linker options. Incompatible options such as \'b_asneeded\' have been disabled.', fatal=False)
             mlog.warning('Please see https://mesonbuild.com/Builtin-options.html#Notes_about_Apple_Bitcode_support for more details.', fatal=False)
 
-class CmdLineFileParser(configparser.ConfigParser):
-    def __init__(self) -> None:
-        # We don't want ':' as key delimiter, otherwise it would break when
-        # storing subproject options like "subproject:option=value"
-        super().__init__(delimiters=['='], interpolation=None)
-
-    def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
-        return super().read(filenames, encoding)
-
-    def optionxform(self, optionstr: str) -> str:
-        # Don't call str.lower() on keys
-        return optionstr
-
-class MachineFileParser():
-    def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
-        self.parser = CmdLineFileParser()
-        self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
-        self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
-
-        for fname in filenames:
-            try:
-                with open(fname, encoding='utf-8') as f:
-                    content = f.read()
-            except UnicodeDecodeError as e:
-                raise EnvironmentException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
-
-            content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
-            content = content.replace('@DIRNAME@', os.path.dirname(fname))
-            try:
-                self.parser.read_string(content, fname)
-            except configparser.Error as e:
-                raise EnvironmentException(f'Malformed machine file: {e}')
-
-        # Parse [constants] first so they can be used in other sections
-        if self.parser.has_section('constants'):
-            self.constants.update(self._parse_section('constants'))
-
-        for s in self.parser.sections():
-            if s == 'constants':
-                continue
-            self.sections[s] = self._parse_section(s)
-
-    def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
-        self.scope = self.constants.copy()
-        section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
-        for entry, value in self.parser.items(s):
-            if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
-                raise EnvironmentException(f'Malformed variable name {entry!r} in machine file.')
-            # Windows paths...
-            value = value.replace('\\', '\\\\')
-            try:
-                ast = mparser.Parser(value, 'machinefile').parse()
-                if not ast.lines:
-                    raise EnvironmentException('value cannot be empty')
-                res = self._evaluate_statement(ast.lines[0])
-            except MesonException as e:
-                raise EnvironmentException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
-            except KeyError as e:
-                raise EnvironmentException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
-            section[entry] = res
-            self.scope[entry] = res
-        return section
-
-    def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
-        if isinstance(node, (mparser.StringNode)):
-            return node.value
-        elif isinstance(node, mparser.BooleanNode):
-            return node.value
-        elif isinstance(node, mparser.NumberNode):
-            return node.value
-        elif isinstance(node, mparser.ParenthesizedNode):
-            return self._evaluate_statement(node.inner)
-        elif isinstance(node, mparser.ArrayNode):
-            # TODO: This is where recursive types would come in handy
-            return [self._evaluate_statement(arg) for arg in node.args.arguments]
-        elif isinstance(node, mparser.IdNode):
-            return self.scope[node.value]
-        elif isinstance(node, mparser.ArithmeticNode):
-            l = self._evaluate_statement(node.left)
-            r = self._evaluate_statement(node.right)
-            if node.operation == 'add':
-                if (isinstance(l, str) and isinstance(r, str)) or \
-                   (isinstance(l, list) and isinstance(r, list)):
-                    return l + r
-            elif node.operation == 'div':
-                if isinstance(l, str) and isinstance(r, str):
-                    return os.path.join(l, r)
-        raise EnvironmentException('Unsupported node type')
-
-def parse_machine_files(filenames: T.List[str], sourcedir: str):
-    parser = MachineFileParser(filenames, sourcedir)
-    return parser.sections
-
 def get_cmd_line_file(build_dir: str) -> str:
     return os.path.join(build_dir, 'meson-private', 'cmd_line.txt')
 

mesonbuild/environment.py

@@ -11,6 +11,10 @@ import collections
 
 from . import coredata
 from . import mesonlib
+from . import machinefile
+
+CmdLineFileParser = machinefile.CmdLineFileParser
+
 from .mesonlib import (
     MesonException, MachineChoice, Popen_safe, PerMachine,
     PerMachineDefaultable, PerThreeMachineDefaultable, split_args, quote_arg, OptionKey,
@@ -589,7 +593,7 @@ class Environment:
         ## Read in native file(s) to override build machine configuration
 
         if self.coredata.config_files is not None:
-            config = coredata.parse_machine_files(self.coredata.config_files, self.source_dir)
+            config = machinefile.parse_machine_files(self.coredata.config_files, self.source_dir)
             binaries.build = BinaryTable(config.get('binaries', {}))
             properties.build = Properties(config.get('properties', {}))
             cmakevars.build = CMakeVariables(config.get('cmake', {}))
@@ -600,7 +604,7 @@ class Environment:
         ## Read in cross file(s) to override host machine configuration
 
         if self.coredata.cross_files:
-            config = coredata.parse_machine_files(self.coredata.cross_files, self.source_dir)
+            config = machinefile.parse_machine_files(self.coredata.cross_files, self.source_dir)
             properties.host = Properties(config.get('properties', {}))
             binaries.host = BinaryTable(config.get('binaries', {}))
             cmakevars.host = CMakeVariables(config.get('cmake', {}))

mesonbuild/machinefile.py

@@ -1,15 +1,117 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2013-2024 Contributors to the The Meson project
 
-from . import mlog
+import typing as T
+import configparser
+import os
+
+from . import mparser
+
+from .mesonlib import MesonException
+
+if T.TYPE_CHECKING:
+    from .compilers import Compiler
+    from .coredata import StrOrBytesPath
+
+    CompilersDict = T.Dict[str, Compiler]
+
+
+class CmdLineFileParser(configparser.ConfigParser):
+    def __init__(self) -> None:
+        # We don't want ':' as key delimiter, otherwise it would break when
+        # storing subproject options like "subproject:option=value"
+        super().__init__(delimiters=['='], interpolation=None)
+
+    def read(self, filenames: T.Union['StrOrBytesPath', T.Iterable['StrOrBytesPath']], encoding: T.Optional[str] = 'utf-8') -> T.List[str]:
+        return super().read(filenames, encoding)
+
+    def optionxform(self, optionstr: str) -> str:
+        # Don't call str.lower() on keys
+        return optionstr
+
+
+class MachineFileParser():
+    def __init__(self, filenames: T.List[str], sourcedir: str) -> None:
+        self.parser = CmdLineFileParser()
+        self.constants: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {'True': True, 'False': False}
+        self.sections: T.Dict[str, T.Dict[str, T.Union[str, bool, int, T.List[str]]]] = {}
+
+        for fname in filenames:
+            try:
+                with open(fname, encoding='utf-8') as f:
+                    content = f.read()
+            except UnicodeDecodeError as e:
+                raise MesonException(f'Malformed machine file {fname!r} failed to parse as unicode: {e}')
+
+            content = content.replace('@GLOBAL_SOURCE_ROOT@', sourcedir)
+            content = content.replace('@DIRNAME@', os.path.dirname(fname))
+            try:
+                self.parser.read_string(content, fname)
+            except configparser.Error as e:
+                raise MesonException(f'Malformed machine file: {e}')
+
+        # Parse [constants] first so they can be used in other sections
+        if self.parser.has_section('constants'):
+            self.constants.update(self._parse_section('constants'))
+
+        for s in self.parser.sections():
+            if s == 'constants':
+                continue
+            self.sections[s] = self._parse_section(s)
+
+    def _parse_section(self, s: str) -> T.Dict[str, T.Union[str, bool, int, T.List[str]]]:
+        self.scope = self.constants.copy()
+        section: T.Dict[str, T.Union[str, bool, int, T.List[str]]] = {}
+        for entry, value in self.parser.items(s):
+            if ' ' in entry or '\t' in entry or "'" in entry or '"' in entry:
+                raise MesonException(f'Malformed variable name {entry!r} in machine file.')
+            # Windows paths...
+            value = value.replace('\\', '\\\\')
+            try:
+                ast = mparser.Parser(value, 'machinefile').parse()
+                if not ast.lines:
+                    raise MesonException('value cannot be empty')
+                res = self._evaluate_statement(ast.lines[0])
+            except MesonException as e:
+                raise MesonException(f'Malformed value in machine file variable {entry!r}: {str(e)}.')
+            except KeyError as e:
+                raise MesonException(f'Undefined constant {e.args[0]!r} in machine file variable {entry!r}.')
+            section[entry] = res
+            self.scope[entry] = res
+        return section
+
+    def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
+        if isinstance(node, (mparser.StringNode)):
+            return node.value
+        elif isinstance(node, mparser.BooleanNode):
+            return node.value
+        elif isinstance(node, mparser.NumberNode):
+            return node.value
+        elif isinstance(node, mparser.ParenthesizedNode):
+            return self._evaluate_statement(node.inner)
+        elif isinstance(node, mparser.ArrayNode):
+            # TODO: This is where recursive types would come in handy
+            return [self._evaluate_statement(arg) for arg in node.args.arguments]
+        elif isinstance(node, mparser.IdNode):
+            return self.scope[node.value]
+        elif isinstance(node, mparser.ArithmeticNode):
+            l = self._evaluate_statement(node.left)
+            r = self._evaluate_statement(node.right)
+            if node.operation == 'add':
+                if (isinstance(l, str) and isinstance(r, str)) or \
+                   (isinstance(l, list) and isinstance(r, list)):
+                    return l + r
+            elif node.operation == 'div':
+                if isinstance(l, str) and isinstance(r, str):
+                    return os.path.join(l, r)
+        raise MesonException('Unsupported node type')
+
+def parse_machine_files(filenames: T.List[str], sourcedir: str):
+    parser = MachineFileParser(filenames, sourcedir)
+    return parser.sections
 
-class MachineFile:
-    def __init__(self, fname):
-        with open(fname, encoding='utf-8') as f:
-            pass
-        self.stuff = None
 
 class MachineFileStore:
-    def __init__(self, native_files, cross_files):
+    def __init__(self, native_files, cross_files, source_dir):
-        self.native = [MachineFile(x) for x in native_files]
+        self.native = MachineFileParser(native_files if native_files is not None else [], source_dir).sections
-        self.cross = [MachineFile(x) for x in cross_files]
+        self.cross = MachineFileParser(cross_files if cross_files is not None else [], source_dir).sections

mesonbuild/scripts/scanbuild.py

@@ -7,7 +7,8 @@ import subprocess
 import shutil
 import tempfile
 from ..environment import detect_ninja, detect_scanbuild
-from ..coredata import get_cmd_line_file, CmdLineFileParser
+from ..coredata import get_cmd_line_file
+from ..machinefile import CmdLineFileParser
 from ..mesonlib import windows_proof_rmtree
 from pathlib import Path
 import typing as T

test cases/unit/116 empty project/expected_mods.json

@@ -217,6 +217,7 @@
       "mesonbuild.linkers",
       "mesonbuild.linkers.base",
       "mesonbuild.linkers.detect",
+      "mesonbuild.machinefile",
       "mesonbuild.mesonlib",
       "mesonbuild.mesonmain",
       "mesonbuild.mintro",

unittests/machinefiletests.py

@@ -59,7 +59,7 @@ class MachineFileStoreTests(TestCase):
 
     def test_loading(self):
         store = machinefile.MachineFileStore([cross_dir / 'ubuntu-armhf.txt'], [], str(cross_dir))
-        self.assertTrue(True)
+        self.assertIsNotNone(store)
 
 class NativeFileTests(BasePlatformTests):
 

unittests/platformagnostictests.py

@@ -274,7 +274,7 @@ class PlatformAgnosticTests(BasePlatformTests):
             expected = json.load(f)['meson']['modules']
 
         self.assertEqual(data['modules'], expected)
-        self.assertEqual(data['count'], 69)
+        self.assertEqual(data['count'], 70)
 
     def test_meson_package_cache_dir(self):
         # Copy testdir into temporary directory to not pollute meson source tree.