From 79ed2415e9a50e5181b203790ec057c6229609a3 Mon Sep 17 00:00:00 2001 From: JCWasmx86 Date: Mon, 31 Oct 2022 16:47:40 +0100 Subject: [PATCH 01/18] Add ParenthesizedNode --- mesonbuild/ast/visitor.py | 4 ++++ mesonbuild/interpreterbase/interpreterbase.py | 2 ++ mesonbuild/mparser.py | 11 ++++++++++- mesonbuild/optinterpreter.py | 2 ++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 8a0e77bb8..51a662024 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -144,3 +144,7 @@ class AstVisitor: for key, val in node.kwargs.items(): key.accept(self) val.accept(self) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + self.visit_default_func(node) + node.inner.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 902f84a45..f35da33bd 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -239,6 +239,8 @@ class InterpreterBase: raise ContinueRequest() elif isinstance(cur, mparser.BreakNode): raise BreakRequest() + elif isinstance(cur, mparser.ParenthesizedNode): + return self.evaluate_statement(cur.inner) elif isinstance(cur, mparser.TestCaseClauseNode): return self.evaluate_testcase(cur) else: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index fb4e43367..3697d8a1a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -577,6 +577,15 @@ class TernaryNode(BaseNode): self.trueblock = trueblock self.falseblock = falseblock +@dataclass(unsafe_hash=True) +class ParenthesizedNode(BaseNode): + + inner: BaseNode + + def __init__(self, inner: BaseNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + self.inner = inner + if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] @@ -778,7 +787,7 @@ class Parser: if self.accept('lparen'): e = self.statement() self.block_expect('rparen', block_start) - return e + return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lbracket'): args = self.args() self.block_expect('rbracket', block_start) diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 8ad84aaa8..37ef55ab0 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -113,6 +113,8 @@ class OptionInterpreter: def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var': if isinstance(arg, str): return arg + if isinstance(arg, mparser.ParenthesizedNode): + return self.reduce_single(arg.inner) elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, mparser.NumberNode)): return arg.value From 3ff3b8abf6f24f676b6c03250ba96a365ae77b03 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Tue, 22 Aug 2023 22:05:40 -0400 Subject: [PATCH 02/18] parser: more specific error for float numbers --- mesonbuild/mparser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 3697d8a1a..9867c7164 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -863,6 +863,9 @@ class Parser: def method_call(self, source_object: BaseNode) -> MethodNode: methodname = self.e9() if not isinstance(methodname, IdNode): + if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode): + raise ParseException('meson does not support float numbers', + self.getline(), source_object.lineno, source_object.colno) raise ParseException('Method name must be plain id', self.getline(), self.current.lineno, self.current.colno) assert isinstance(methodname.value, str) From 5707d390174b7e8db431bc9cee2257043ea59f84 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 09:32:21 -0400 Subject: [PATCH 03/18] parser: preserve number base --- mesonbuild/cargo/builder.py | 2 +- mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/mparser.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index fb086d164..2442fbfa7 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -45,7 +45,7 @@ def number(value: int, filename: str) -> mparser.NumberNode: :param filename: the file that the value came from :return: A NumberNode """ - return mparser.NumberNode(_token('number', filename, value)) + return mparser.NumberNode(_token('number', filename, str(value))) def bool(value: builtins.bool, filename: str) -> mparser.BooleanNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index f88d091ab..f577ba774 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -966,7 +966,7 @@ class CMakeInterpreter: return IdNode(token(val=value)) def number(value: int) -> NumberNode: - return NumberNode(token(val=value)) + return NumberNode(token(val=str(value))) def nodeify(value: TYPE_mixed_list) -> BaseNode: if isinstance(value, str): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 9867c7164..5ab76e7dc 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -217,7 +217,7 @@ class Lexer: lineno += len(lines) - 1 line_start = mo.end() - len(lines[-1]) elif tid == 'number': - value = int(match_text, base=0) + value = match_text elif tid == 'eol_cont': lineno += 1 line_start = loc @@ -285,8 +285,16 @@ class IdNode(ElementaryNode[str]): def __str__(self) -> str: return "Id node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) +@dataclass(unsafe_hash=True) class NumberNode(ElementaryNode[int]): - pass + + raw_value: str = field(hash=False) + + def __init__(self, token: Token[str]): + BaseNode.__init__(self, token.lineno, token.colno, token.filename) + self.raw_value = token.value + self.value = int(token.value, base=0) + self.bytespan = token.bytespan class StringNode(ElementaryNode[str]): def __str__(self) -> str: From a730a2fe215ae45c928370b5e28d2a844c082f38 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:51:58 -0400 Subject: [PATCH 04/18] parser: remove useless __str__ methods on nodes --- mesonbuild/mparser.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 5ab76e7dc..f9b896759 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -282,8 +282,7 @@ class BooleanNode(ElementaryNode[bool]): pass class IdNode(ElementaryNode[str]): - def __str__(self) -> str: - return "Id node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) + pass @dataclass(unsafe_hash=True) class NumberNode(ElementaryNode[int]): @@ -297,16 +296,13 @@ class NumberNode(ElementaryNode[int]): self.bytespan = token.bytespan class StringNode(ElementaryNode[str]): - def __str__(self) -> str: - return "String node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno) + pass class FormatStringNode(ElementaryNode[str]): - def __str__(self) -> str: - return f"Format string node: '{self.value}' ({self.lineno}, {self.colno})." + pass class MultilineFormatStringNode(FormatStringNode): - def __str__(self) -> str: - return f"Multiline Format string node: '{self.value}' ({self.lineno}, {self.colno})." + pass class ContinueNode(ElementaryNode): pass From 35936283d24ed5a0aa76b184a7489d637d3e49c4 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 09:24:05 -0400 Subject: [PATCH 05/18] parser: preserve escape chars in strings use separate Node for multiline strings --- mesonbuild/ast/interpreter.py | 2 +- mesonbuild/ast/introspection.py | 8 ++--- mesonbuild/ast/printer.py | 12 ++++++- mesonbuild/ast/visitor.py | 6 ++++ mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/coredata.py | 2 +- mesonbuild/interpreter/interpreter.py | 2 +- mesonbuild/interpreterbase/helpers.py | 4 +-- mesonbuild/interpreterbase/interpreterbase.py | 20 +++++------ mesonbuild/mintro.py | 4 +-- mesonbuild/mparser.py | 33 ++++++++++++------- mesonbuild/optinterpreter.py | 4 +-- mesonbuild/rewriter.py | 24 +++++++------- 13 files changed, 74 insertions(+), 49 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 9e098d0be..501be7dfe 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -239,7 +239,7 @@ class AstInterpreter(InterpreterBase): def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs: def resolve_key(node: mparser.BaseNode) -> str: - if isinstance(node, mparser.StringNode): + if isinstance(node, mparser.BaseStringNode): return node.value return '__AST_UNKNOWN__' arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key) diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index e8055c529..50aa7444c 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -27,7 +27,7 @@ from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary from ..compilers import detect_compiler_for from ..interpreterbase import InvalidArguments from ..mesonlib import MachineChoice, OptionKey -from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode +from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode from .interpreter import AstInterpreter if T.TYPE_CHECKING: @@ -128,7 +128,7 @@ class IntrospectionInterpreter(AstInterpreter): if not self.is_subproject() and 'subproject_dir' in kwargs: spdirname = kwargs['subproject_dir'] - if isinstance(spdirname, StringNode): + if isinstance(spdirname, BaseStringNode): assert isinstance(spdirname.value, str) self.subproject_dir = spdirname.value if not self.is_subproject(): @@ -174,7 +174,7 @@ class IntrospectionInterpreter(AstInterpreter): for l in self.flatten_args(raw_langs): if isinstance(l, str): langs.append(l) - elif isinstance(l, StringNode): + elif isinstance(l, BaseStringNode): langs.append(l.value) for lang in sorted(langs, key=compilers.sort_clink): @@ -263,7 +263,7 @@ class IntrospectionInterpreter(AstInterpreter): # Pop the first element if the function is a build target function if isinstance(curr, FunctionNode) and curr.func_name in BUILD_TARGET_FUNCTIONS: arg_nodes.pop(0) - elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))] + elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] if elementary_nodes: res += [curr] diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index ebf63afeb..4626e5eed 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -84,7 +84,17 @@ class AstPrinter(AstVisitor): def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: assert isinstance(node.value, str) - self.append("f'" + node.value + "'", node) + self.append("f'" + self.escape(node.value) + "'", node) + node.lineno = self.curr_line or node.lineno + + def visit_MultilineStringNode(self, node: mparser.StringNode) -> None: + assert isinstance(node.value, str) + self.append("'''" + node.value + "'''", node) + node.lineno = self.curr_line or node.lineno + + def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + assert isinstance(node.value, str) + self.append("f'''" + node.value + "'''", node) node.lineno = self.curr_line or node.lineno def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 51a662024..69b93c7f0 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -43,6 +43,12 @@ class AstVisitor: def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: self.visit_default_func(node) + def visit_MultilineStringNode(self, node: mparser.StringNode) -> None: + self.visit_default_func(node) + + def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None: + self.visit_default_func(node) + def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: self.visit_default_func(node) diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index f577ba774..80dfff50a 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -960,7 +960,7 @@ class CMakeInterpreter: return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) def string(value: str) -> StringNode: - return StringNode(token(val=value)) + return StringNode(token(val=value), escape=False) def id_node(value: str) -> IdNode: return IdNode(token(val=value)) diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py index f151c7b81..f91c5837f 100644 --- a/mesonbuild/coredata.py +++ b/mesonbuild/coredata.py @@ -1098,7 +1098,7 @@ class MachineFileParser(): return section def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: - if isinstance(node, (mparser.StringNode)): + if isinstance(node, (mparser.BaseStringNode)): return node.value elif isinstance(node, mparser.BooleanNode): return node.value diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 9b005cb78..0814739f5 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -536,7 +536,7 @@ class Interpreter(InterpreterBase, HoldableObject): assert isinstance(kw, mparser.IdNode), 'for mypy' if kw.value == 'meson_version': # mypy does not understand "and isinstance" - if isinstance(val, mparser.StringNode): + if isinstance(val, mparser.BaseStringNode): self.handle_meson_version(val.value, val) def get_build_def_files(self) -> mesonlib.OrderedSet[str]: diff --git a/mesonbuild/interpreterbase/helpers.py b/mesonbuild/interpreterbase/helpers.py index f2ee1b1a9..917969b26 100644 --- a/mesonbuild/interpreterbase/helpers.py +++ b/mesonbuild/interpreterbase/helpers.py @@ -25,7 +25,7 @@ if T.TYPE_CHECKING: from .baseobjects import TYPE_var, TYPE_kwargs, SubProject def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']: - if isinstance(args, mparser.StringNode): + if isinstance(args, mparser.BaseStringNode): assert isinstance(args.value, str) return [args.value] if not isinstance(args, collections.abc.Sequence): @@ -35,7 +35,7 @@ def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var'] if isinstance(a, list): rest = flatten(a) result = result + rest - elif isinstance(a, mparser.StringNode): + elif isinstance(a, mparser.BaseStringNode): result.append(a.value) else: result.append(a) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index f35da33bd..eab96efb1 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -196,8 +196,13 @@ class InterpreterBase: self.assignment(cur) elif isinstance(cur, mparser.MethodNode): return self.method_call(cur) - elif isinstance(cur, mparser.StringNode): - return self._holderify(cur.value) + elif isinstance(cur, mparser.BaseStringNode): + if isinstance(cur, mparser.MultilineFormatStringNode): + return self.evaluate_multiline_fstring(cur) + elif isinstance(cur, mparser.FormatStringNode): + return self.evaluate_fstring(cur) + else: + return self._holderify(cur.value) elif isinstance(cur, mparser.BooleanNode): return self._holderify(cur.value) elif isinstance(cur, mparser.IfClauseNode): @@ -230,11 +235,6 @@ class InterpreterBase: return self.evaluate_indexing(cur) elif isinstance(cur, mparser.TernaryNode): return self.evaluate_ternary(cur) - elif isinstance(cur, mparser.FormatStringNode): - if isinstance(cur, mparser.MultilineFormatStringNode): - return self.evaluate_multiline_fstring(cur) - else: - return self.evaluate_fstring(cur) elif isinstance(cur, mparser.ContinueNode): raise ContinueRequest() elif isinstance(cur, mparser.BreakNode): @@ -256,7 +256,7 @@ class InterpreterBase: @FeatureNew('dict', '0.47.0') def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject: def resolve_key(key: mparser.BaseNode) -> str: - if not isinstance(key, mparser.StringNode): + if not isinstance(key, mparser.BaseStringNode): FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject) key_holder = self.evaluate_statement(key) if key_holder is None: @@ -428,9 +428,7 @@ class InterpreterBase: return self.evaluate_fstring(node) @FeatureNew('format strings', '0.58.0') - def evaluate_fstring(self, node: mparser.FormatStringNode) -> InterpreterObject: - assert isinstance(node, mparser.FormatStringNode) - + def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> InterpreterObject: def replace(match: T.Match[str]) -> str: var = str(match.group(1)) try: diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index 039153532..cf4a53ab0 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -36,7 +36,7 @@ from .dependencies import Dependency from . import environment from .interpreterbase import ObjectHolder from .mesonlib import OptionKey -from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode +from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode if T.TYPE_CHECKING: import argparse @@ -194,7 +194,7 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st elif isinstance(n, ArgumentNode): args = n.arguments for j in args: - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): assert isinstance(j.value, str) res += [Path(j.value)] elif isinstance(j, str): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index f9b896759..614479466 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -193,23 +193,15 @@ class Lexer: elif tid == 'dblquote': raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col) elif tid in {'string', 'fstring'}: - # Handle here and not on the regexp to give a better error message. if match_text.find("\n") != -1: msg = ("Newline character in a string detected, use ''' (three single quotes) " "for multiline strings instead.\n" "This will become a hard error in a future Meson release.") mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename)) value = match_text[2 if tid == 'fstring' else 1:-1] - value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, value) elif tid in {'multiline_string', 'multiline_fstring'}: - # For multiline strings, parse out the value and pass - # through the normal string logic. - # For multiline format strings, we have to emit a - # different AST node so we can add a feature check, - # but otherwise, it follows the normal fstring logic. if tid == 'multiline_string': value = match_text[3:-3] - tid = 'string' else: value = match_text[4:-3] lines = match_text.split('\n') @@ -295,13 +287,30 @@ class NumberNode(ElementaryNode[int]): self.value = int(token.value, base=0) self.bytespan = token.bytespan -class StringNode(ElementaryNode[str]): +class BaseStringNode(ElementaryNode[str]): pass -class FormatStringNode(ElementaryNode[str]): +@dataclass(unsafe_hash=True) +class StringNode(BaseStringNode): + + raw_value: str = field(hash=False) + + def __init__(self, token: Token[str], escape: bool = True): + super().__init__(token) + self.value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, token.value) if escape else token.value + self.raw_value = token.value + +class FormatStringNode(StringNode): pass -class MultilineFormatStringNode(FormatStringNode): +@dataclass(unsafe_hash=True) +class MultilineStringNode(BaseStringNode): + + def __init__(self, token: Token[str]): + super().__init__(token) + self.value = token.value + +class MultilineFormatStringNode(MultilineStringNode): pass class ContinueNode(ElementaryNode): @@ -819,6 +828,8 @@ class Parser: return StringNode(t) if self.accept('fstring'): return FormatStringNode(t) + if self.accept('multiline_string'): + return MultilineStringNode(t) if self.accept('multiline_fstring'): return MultilineFormatStringNode(t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 37ef55ab0..9eeb3fdc6 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -115,7 +115,7 @@ class OptionInterpreter: return arg if isinstance(arg, mparser.ParenthesizedNode): return self.reduce_single(arg.inner) - elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, + elif isinstance(arg, (mparser.BaseStringNode, mparser.BooleanNode, mparser.NumberNode)): return arg.value elif isinstance(arg, mparser.ArrayNode): @@ -123,7 +123,7 @@ class OptionInterpreter: elif isinstance(arg, mparser.DictNode): d = {} for k, v in arg.args.kwargs.items(): - if not isinstance(k, mparser.StringNode): + if not isinstance(k, mparser.BaseStringNode): raise OptionException('Dictionary keys must be a string literal') d[k.value] = self.reduce_single(v) return d diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index a9b2e881c..0d7b111c4 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode import json, os, re, sys import typing as T @@ -267,12 +267,12 @@ class MTypeStrList(MTypeList): return StringNode(Token('', '', 0, 0, 0, None, str(value))) def _check_is_equal(self, node, value) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return node.value == value return False def _check_regex_matches(self, node, regex: str) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False @@ -292,7 +292,7 @@ class MTypeIDList(MTypeList): return False def _check_regex_matches(self, node, regex: str) -> bool: - if isinstance(node, StringNode): + if isinstance(node, BaseStringNode): return re.match(regex, node.value) is not None return False @@ -652,7 +652,7 @@ class Rewriter: src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): src_list += [j.value] # Generate the new String nodes @@ -686,7 +686,7 @@ class Rewriter: def find_node(src): for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): if j.value == src: return i, j return None, None @@ -745,7 +745,7 @@ class Rewriter: extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): extra_files_list += [j.value] # Generate the new String nodes @@ -776,7 +776,7 @@ class Rewriter: def find_node(src): for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): if j.value == src: return i, j return None, None @@ -845,12 +845,12 @@ class Rewriter: src_list = [] for i in target['sources']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): src_list += [j.value] extra_files_list = [] for i in target['extra_files']: for j in arg_list_from_node(i): - if isinstance(j, StringNode): + if isinstance(j, BaseStringNode): extra_files_list += [j.value] test_data = { 'name': target['name'], @@ -865,8 +865,8 @@ class Rewriter: alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))]) - unknown = [x for x in i.arguments if not isinstance(x, StringNode)] - sources = [x for x in i.arguments if isinstance(x, StringNode)] + unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)] + sources = [x for x in i.arguments if isinstance(x, BaseStringNode)] sources = sorted(sources, key=lambda x: path_sorter(x.value)) i.arguments = unknown + sources From 306562b4666dea7828af680afbd4d111e667027f Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 14:22:15 -0400 Subject: [PATCH 06/18] parser: use IdNode for function name and assignment name --- mesonbuild/ast/interpreter.py | 19 +++++++------ mesonbuild/ast/introspection.py | 2 +- mesonbuild/ast/printer.py | 16 +++++------ mesonbuild/ast/visitor.py | 4 +++ mesonbuild/cargo/builder.py | 6 ++-- mesonbuild/cmake/interpreter.py | 6 ++-- mesonbuild/interpreter/interpreter.py | 4 +-- mesonbuild/interpreterbase/interpreterbase.py | 10 +++---- mesonbuild/mintro.py | 2 +- mesonbuild/mparser.py | 28 ++++++++----------- mesonbuild/optinterpreter.py | 2 +- mesonbuild/rewriter.py | 14 +++++----- 12 files changed, 57 insertions(+), 56 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 501be7dfe..1dea817b1 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -254,10 +254,10 @@ class AstInterpreter(InterpreterBase): def evaluate_plusassign(self, node: PlusAssignmentNode) -> None: assert isinstance(node, PlusAssignmentNode) # Cheat by doing a reassignment - self.assignments[node.var_name] = node.value # Save a reference to the value node + self.assignments[node.var_name.value] = node.value # Save a reference to the value node if node.value.ast_id: self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name] = self.evaluate_statement(node.value) + self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) def evaluate_indexing(self, node: IndexNode) -> int: return 0 @@ -319,10 +319,10 @@ class AstInterpreter(InterpreterBase): def assignment(self, node: AssignmentNode) -> None: assert isinstance(node, AssignmentNode) - self.assignments[node.var_name] = node.value # Save a reference to the value node + self.assignments[node.var_name.value] = node.value # Save a reference to the value node if node.value.ast_id: self.reverse_assignment[node.value.ast_id] = node - self.assign_vals[node.var_name] = self.evaluate_statement(node.value) # Evaluate the value just in case + self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) # Evaluate the value just in case def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]: def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any: @@ -382,17 +382,18 @@ class AstInterpreter(InterpreterBase): src = quick_resolve(node.source_object) margs = self.flatten_args(node.args.arguments, include_unknown_args, id_loop_detect) mkwargs: T.Dict[str, TYPE_nvar] = {} + method_name = node.name.value try: if isinstance(src, str): - result = StringHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = StringHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, bool): - result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, int): - result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, list): - result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) elif isinstance(src, dict): - result = DictHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) + result = DictHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs) except mesonlib.MesonException: return None diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py index 50aa7444c..fa4b8d302 100644 --- a/mesonbuild/ast/introspection.py +++ b/mesonbuild/ast/introspection.py @@ -261,7 +261,7 @@ class IntrospectionInterpreter(AstInterpreter): continue arg_nodes = arg_node.arguments.copy() # Pop the first element if the function is a build target function - if isinstance(curr, FunctionNode) and curr.func_name in BUILD_TARGET_FUNCTIONS: + if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS: arg_nodes.pop(0) elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 4626e5eed..020299d6b 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -162,24 +162,24 @@ class AstPrinter(AstVisitor): def visit_MethodNode(self, node: mparser.MethodNode) -> None: node.lineno = self.curr_line or node.lineno node.source_object.accept(self) - self.append('.' + node.name + '(', node) + self.append('.' + node.name.value + '(', node) node.args.accept(self) self.append(')', node) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.func_name + '(', node) + self.append(node.func_name.value + '(', node) node.args.accept(self) self.append(')', node) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.var_name + ' = ', node) + self.append(node.var_name.value + ' = ', node) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: node.lineno = self.curr_line or node.lineno - self.append(node.var_name + ' += ', node) + self.append(node.var_name.value + ' += ', node) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: @@ -352,22 +352,22 @@ class AstJSONPrinter(AstVisitor): def visit_MethodNode(self, node: mparser.MethodNode) -> None: self._accept('object', node.source_object) self._accept('args', node.args) - self.current['name'] = node.name + self.current['name'] = node.name.value self.setbase(node) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: self._accept('args', node.args) - self.current['name'] = node.func_name + self.current['name'] = node.func_name.value self.setbase(node) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: self._accept('value', node.value) - self.current['var_name'] = node.var_name + self.current['var_name'] = node.var_name.value self.setbase(node) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: self._accept('value', node.value) - self.current['var_name'] = node.var_name + self.current['var_name'] = node.var_name.value self.setbase(node) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 69b93c7f0..6e40ea0b4 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -103,18 +103,22 @@ class AstVisitor: def visit_MethodNode(self, node: mparser.MethodNode) -> None: self.visit_default_func(node) node.source_object.accept(self) + node.name.accept(self) node.args.accept(self) def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: self.visit_default_func(node) + node.func_name.accept(self) node.args.accept(self) def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: self.visit_default_func(node) + node.var_name.accept(self) node.value.accept(self) def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: self.visit_default_func(node) + node.var_name.accept(self) node.value.accept(self) def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 2442fbfa7..6ef66eeca 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -97,7 +97,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, name, args) + return mparser.MethodNode(id_.filename, -1, -1, id_, identifier(name, id_.filename), args) def function(name: str, filename: str, @@ -117,7 +117,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, name, args) + return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), args) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: @@ -168,7 +168,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, varname, value) + return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 80dfff50a..0703c1b93 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -1002,7 +1002,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, name, args_n) + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), args_n) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1014,10 +1014,10 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, name, args_n) + return MethodNode(self.subdir.as_posix(), 0, 0, obj, id_node(name), args_n) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, var_name, value) + return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 0814739f5..c6370870c 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -2942,7 +2942,7 @@ class Interpreter(InterpreterBase, HoldableObject): def _add_global_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: if self.is_subproject(): - msg = f'Function \'{node.func_name}\' cannot be used in subprojects because ' \ + msg = f'Function \'{node.func_name.value}\' cannot be used in subprojects because ' \ 'there is no way to make that reliable.\nPlease only call ' \ 'this if is_subproject() returns false. Alternatively, ' \ 'define a variable that\ncontains your language-specific ' \ @@ -2962,7 +2962,7 @@ class Interpreter(InterpreterBase, HoldableObject): def _add_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], args_frozen: bool, args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: if args_frozen: - msg = f'Tried to use \'{node.func_name}\' after a build target has been declared.\n' \ + msg = f'Tried to use \'{node.func_name.value}\' after a build target has been declared.\n' \ 'This is not permitted. Please declare all arguments before your targets.' raise InvalidCode(msg) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index eab96efb1..1c0912835 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -137,7 +137,7 @@ class InterpreterBase: if not self.ast.lines: raise InvalidCode('No statements in code.') first = self.ast.lines[0] - if not isinstance(first, mparser.FunctionNode) or first.func_name != 'project': + if not isinstance(first, mparser.FunctionNode) or first.func_name.value != 'project': p = pathlib.Path(self.source_root).resolve() found = p for parent in p.parents: @@ -476,7 +476,7 @@ class InterpreterBase: def evaluate_plusassign(self, node: mparser.PlusAssignmentNode) -> None: assert isinstance(node, mparser.PlusAssignmentNode) - varname = node.var_name + varname = node.var_name.value addition = self.evaluate_statement(node.value) if addition is None: raise InvalidCodeOnVoid('plus assign') @@ -504,7 +504,7 @@ class InterpreterBase: return self._holderify(iobject.operator_call(MesonOperator.INDEX, index)) def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]: - func_name = node.func_name + func_name = node.func_name.value (h_posargs, h_kwargs) = self.reduce_arguments(node.args) (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs) if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}: @@ -532,7 +532,7 @@ class InterpreterBase: else: object_display_name = invocable.__class__.__name__ obj = self.evaluate_statement(invocable) - method_name = node.name + method_name = node.name.value (h_args, h_kwargs) = self.reduce_arguments(node.args) (args, kwargs) = self._unholder_args(h_args, h_kwargs) if is_disabled(args, kwargs): @@ -628,7 +628,7 @@ class InterpreterBase: Tried to assign values inside an argument list. To specify a keyword argument, use : instead of =. ''')) - var_name = node.var_name + var_name = node.var_name.value if not isinstance(var_name, str): raise InvalidArguments('Tried to assign value to a non-variable.') value = self.evaluate_statement(node.value) diff --git a/mesonbuild/mintro.py b/mesonbuild/mintro.py index cf4a53ab0..7bd420510 100644 --- a/mesonbuild/mintro.py +++ b/mesonbuild/mintro.py @@ -187,7 +187,7 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st args: T.List[BaseNode] = [] if isinstance(n, FunctionNode): args = list(n.args.arguments) - if n.func_name in BUILD_TARGET_FUNCTIONS: + if n.func_name.value in BUILD_TARGET_FUNCTIONS: args.pop(0) elif isinstance(n, ArrayNode): args = n.args.arguments diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 614479466..ceddf0a52 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -470,52 +470,48 @@ class IndexNode(BaseNode): class MethodNode(BaseNode): source_object: BaseNode - name: str + name: IdNode args: ArgumentNode - def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: str, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: IdNode, args: ArgumentNode): super().__init__(lineno, colno, filename) self.source_object = source_object self.name = name - assert isinstance(self.name, str) self.args = args @dataclass(unsafe_hash=True) class FunctionNode(BaseNode): - func_name: str + func_name: IdNode args: ArgumentNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: str, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: IdNode, args: ArgumentNode): super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) self.func_name = func_name - assert isinstance(func_name, str) self.args = args @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): - var_name: str + var_name: IdNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name - assert isinstance(var_name, str) self.value = value @dataclass(unsafe_hash=True) class PlusAssignmentNode(BaseNode): - var_name: str + var_name: IdNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name - assert isinstance(var_name, str) self.value = value @@ -687,14 +683,14 @@ class Parser: if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return PlusAssignmentNode(left.filename, left.lineno, left.colno, left.value, value) + return PlusAssignmentNode(left.filename, left.lineno, left.colno, left, value) elif self.accept('assign'): value = self.e1() if not isinstance(left, IdNode): raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return AssignmentNode(left.filename, left.lineno, left.colno, left.value, value) + return AssignmentNode(left.filename, left.lineno, left.colno, left, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', @@ -783,7 +779,7 @@ class Parser: raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left.value, args) + left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, args) go_again = True while go_again: go_again = False @@ -887,7 +883,7 @@ class Parser: self.expect('lparen') args = self.args() self.expect('rparen') - method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname.value, args) + method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname, args) if self.accept('dot'): return self.method_call(method) return method diff --git a/mesonbuild/optinterpreter.py b/mesonbuild/optinterpreter.py index 9eeb3fdc6..895cada54 100644 --- a/mesonbuild/optinterpreter.py +++ b/mesonbuild/optinterpreter.py @@ -164,7 +164,7 @@ class OptionInterpreter: def evaluate_statement(self, node: mparser.BaseNode) -> None: if not isinstance(node, mparser.FunctionNode): raise OptionException('Option file may only contain option definitions') - func_name = node.func_name + func_name = node.func_name.value if func_name != 'option': raise OptionException('Only calls to option() are allowed in option files.') (posargs, kwargs) = self.reduce_arguments(node.args) diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 0d7b111c4..af3210cf3 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -420,7 +420,7 @@ class Rewriter: if target in self.interpreter.assignments: node = self.interpreter.assignments[target] if isinstance(node, FunctionNode): - if node.func_name in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: + if node.func_name.value in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: tgt = self.interpreter.assign_vals[target] return tgt @@ -440,7 +440,7 @@ class Rewriter: if dependency in self.interpreter.assignments: node = self.interpreter.assignments[dependency] if isinstance(node, FunctionNode): - if node.func_name == 'dependency': + if node.func_name.value == 'dependency': name = self.interpreter.flatten_args(node.args)[0] dep = check_list(name) @@ -630,7 +630,7 @@ class Rewriter: args = [] if isinstance(n, FunctionNode): args = list(n.args.arguments) - if n.func_name in BUILD_TARGET_FUNCTIONS: + if n.func_name.value in BUILD_TARGET_FUNCTIONS: args.pop(0) elif isinstance(n, ArrayNode): args = n.args.arguments @@ -814,15 +814,15 @@ class Rewriter: src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, 'files', src_far_node) - src_ass_node = AssignmentNode(filename, 0, 0, source_id, src_fun_node) + src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), src_far_node) + src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, cmd['target_type'], tgt_arg_node) - tgt_ass_node = AssignmentNode(filename, 0, 0, target_id, tgt_fun_node) + tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), tgt_arg_node) + tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) From 8d357ba62cba0c5ddf3265743454b0ddcb08d83f Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 14:28:40 -0400 Subject: [PATCH 07/18] parser: use IdNode for foreach varnames --- mesonbuild/ast/printer.py | 4 ++-- mesonbuild/ast/visitor.py | 2 ++ mesonbuild/interpreterbase/interpreterbase.py | 4 ++-- mesonbuild/mparser.py | 8 ++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 020299d6b..410aabd46 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -185,7 +185,7 @@ class AstPrinter(AstVisitor): def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: node.lineno = self.curr_line or node.lineno self.append_padded('foreach', node) - self.append_padded(', '.join(node.varnames), node) + self.append_padded(', '.join(varname.value for varname in node.varnames), node) self.append_padded(':', node) node.items.accept(self) self.newline() @@ -373,7 +373,7 @@ class AstJSONPrinter(AstVisitor): def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: self._accept('items', node.items) self._accept('block', node.block) - self.current['varnames'] = node.varnames + self.current['varnames'] = [varname.value for varname in node.varnames] self.setbase(node) def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 6e40ea0b4..5b1fb6a09 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -123,6 +123,8 @@ class AstVisitor: def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: self.visit_default_func(node) + for varname in node.varnames: + varname.accept(self) node.items.accept(self) node.block.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 1c0912835..7e3f8da5a 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -459,14 +459,14 @@ class InterpreterBase: if tsize is None: if isinstance(i, tuple): raise mesonlib.MesonBugException(f'Iteration of {items} returned a tuple even though iter_tuple_size() is None') - self.set_variable(node.varnames[0], self._holderify(i)) + self.set_variable(node.varnames[0].value, self._holderify(i)) else: if not isinstance(i, tuple): raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') if len(i) != tsize: raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') for j in range(tsize): - self.set_variable(node.varnames[j], self._holderify(i[j])) + self.set_variable(node.varnames[j].value, self._holderify(i[j])) try: self.evaluate_codeblock(node.block) except ContinueRequest: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ceddf0a52..2d7f11505 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -518,11 +518,11 @@ class PlusAssignmentNode(BaseNode): @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): - varnames: T.List[str] = field(hash=False) + varnames: T.List[IdNode] = field(hash=False) items: BaseNode block: CodeBlockNode - def __init__(self, token: Token, varnames: T.List[str], items: BaseNode, block: CodeBlockNode): + def __init__(self, token: Token, varnames: T.List[IdNode], items: BaseNode, block: CodeBlockNode): super().__init__(token.lineno, token.colno, token.filename) self.varnames = varnames self.items = items @@ -898,13 +898,13 @@ class Parser: self.expect('id') assert isinstance(t.value, str) varname = t - varnames = [t.value] + varnames = [IdNode(t)] if self.accept('comma'): t = self.current self.expect('id') assert isinstance(t.value, str) - varnames.append(t.value) + varnames.append(IdNode(t)) self.expect('colon') items = self.statement() From b94167ef50e891355a7d9a0014e6b0cf4ea1302d Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:10:07 -0400 Subject: [PATCH 08/18] parser: preserve value of all tokens --- mesonbuild/mparser.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 2d7f11505..2ff32dc28 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -163,7 +163,7 @@ class Lexer: col = 0 while loc < len(self.code): matched = False - value: T.Union[str, bool, int] = None + value: str = '' for (tid, reg) in self.token_specification: mo = reg.match(self.code, loc) if mo: @@ -175,7 +175,7 @@ class Lexer: loc = mo.end() span_end = loc bytespan = (span_start, span_end) - match_text = mo.group() + value = mo.group() if tid in {'ignore', 'comment'}: break elif tid == 'lparen': @@ -193,23 +193,18 @@ class Lexer: elif tid == 'dblquote': raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col) elif tid in {'string', 'fstring'}: - if match_text.find("\n") != -1: + if value.find("\n") != -1: msg = ("Newline character in a string detected, use ''' (three single quotes) " "for multiline strings instead.\n" "This will become a hard error in a future Meson release.") mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename)) - value = match_text[2 if tid == 'fstring' else 1:-1] + value = value[2 if tid == 'fstring' else 1:-1] elif tid in {'multiline_string', 'multiline_fstring'}: - if tid == 'multiline_string': - value = match_text[3:-3] - else: - value = match_text[4:-3] - lines = match_text.split('\n') + value = value[4 if tid == 'multiline_fstring' else 3:-3] + lines = value.split('\n') if len(lines) > 1: lineno += len(lines) - 1 line_start = mo.end() - len(lines[-1]) - elif tid == 'number': - value = match_text elif tid == 'eol_cont': lineno += 1 line_start = loc @@ -220,13 +215,12 @@ class Lexer: if par_count > 0 or bracket_count > 0 or curl_count > 0: break elif tid == 'id': - if match_text in self.keywords: - tid = match_text + if value in self.keywords: + tid = value else: - if match_text in self.future_keywords: - mlog.warning(f"Identifier '{match_text}' will become a reserved keyword in a future release. Please rename it.", + if value in self.future_keywords: + mlog.warning(f"Identifier '{value}' will become a reserved keyword in a future release. Please rename it.", location=BaseNode(lineno, col, filename)) - value = match_text yield Token(tid, filename, curline_start, curline, col, bytespan, value) break if not matched: From 4b7a56caa265c54a9b77381e2db1819ed87f21de Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 08:58:23 -0400 Subject: [PATCH 09/18] parser: remember previous Token --- mesonbuild/mparser.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 2ff32dc28..ce83cc283 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -623,10 +623,12 @@ class Parser: self.lexer = Lexer(code) self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) + self.previous = self.current self.getsym() self.in_ternary = False def getsym(self) -> None: + self.previous = self.current try: self.current = next(self.stream) except StopIteration: @@ -831,10 +833,9 @@ class Parser: while not isinstance(s, EmptyNode): if self.accept('colon'): a.set_kwarg_no_check(s, self.statement()) - potential = self.current if not self.accept('comma'): return a - a.commas.append(potential) + a.commas.append(self.previous) else: raise ParseException('Only key:value pairs are valid in dict construction.', self.getline(), s.lineno, s.colno) @@ -846,19 +847,17 @@ class Parser: a = ArgumentNode(self.current) while not isinstance(s, EmptyNode): - potential = self.current if self.accept('comma'): - a.commas.append(potential) + a.commas.append(self.previous) a.append(s) elif self.accept('colon'): if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) a.set_kwarg(s, self.statement()) - potential = self.current if not self.accept('comma'): return a - a.commas.append(potential) + a.commas.append(self.previous) else: a.append(s) return a @@ -888,17 +887,15 @@ class Parser: return IndexNode(source_object, index_statement) def foreachblock(self) -> ForeachClauseNode: - t = self.current self.expect('id') - assert isinstance(t.value, str) - varname = t - varnames = [IdNode(t)] + assert isinstance(self.previous.value, str) + varname = self.previous + varnames = [IdNode(self.previous)] if self.accept('comma'): - t = self.current self.expect('id') - assert isinstance(t.value, str) - varnames.append(IdNode(t)) + assert isinstance(self.previous.value, str) + varnames.append(IdNode(self.previous)) self.expect('colon') items = self.statement() From f13260dd43771bd7e9594cbad942b7d9aa4292fc Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:18:21 -0400 Subject: [PATCH 10/18] parser: add ElseNode --- mesonbuild/ast/interpreter.py | 2 +- mesonbuild/ast/visitor.py | 4 ++++ mesonbuild/interpreterbase/interpreterbase.py | 2 +- mesonbuild/mparser.py | 17 +++++++++++++---- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index 1dea817b1..da2119c89 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -312,7 +312,7 @@ class AstInterpreter(InterpreterBase): for i in node.ifs: self.evaluate_codeblock(i.block) if not isinstance(node.elseblock, EmptyNode): - self.evaluate_codeblock(node.elseblock) + self.evaluate_codeblock(node.elseblock.block) def get_variable(self, varname: str) -> int: return 0 diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index 5b1fb6a09..a83782655 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -143,6 +143,10 @@ class AstVisitor: node.condition.accept(self) node.block.accept(self) + def visit_ElseNode(self, node: mparser.IfNode) -> None: + self.visit_default_func(node) + node.block.accept(self) + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: self.visit_default_func(node) node.condition.accept(self) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 7e3f8da5a..5b07dc075 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -303,7 +303,7 @@ class InterpreterBase: mesonlib.project_meson_versions[self.subproject] = prev_meson_version return None if not isinstance(node.elseblock, mparser.EmptyNode): - self.evaluate_codeblock(node.elseblock) + self.evaluate_codeblock(node.elseblock.block) return None def evaluate_testcase(self, node: mparser.TestCaseClauseNode) -> T.Optional[Disabler]: diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ce83cc283..6b578b5af 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -534,17 +534,25 @@ class IfNode(BaseNode): self.condition = condition self.block = block +@dataclass(unsafe_hash=True) +class ElseNode(BaseNode): + + block: CodeBlockNode + + def __init__(self, block: CodeBlockNode): + super().__init__(block.lineno, block.colno, block.filename) + self.block = block @dataclass(unsafe_hash=True) class IfClauseNode(BaseNode): ifs: T.List[IfNode] = field(hash=False) - elseblock: T.Union[EmptyNode, CodeBlockNode] + elseblock: T.Union[EmptyNode, ElseNode] def __init__(self, linenode: BaseNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) self.ifs = [] - self.elseblock = None + self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename) @dataclass(unsafe_hash=True) class TestCaseClauseNode(BaseNode): @@ -919,10 +927,11 @@ class Parser: b = self.codeblock() clause.ifs.append(IfNode(s, s, b)) - def elseblock(self) -> T.Union[CodeBlockNode, EmptyNode]: + def elseblock(self) -> T.Union[ElseNode, EmptyNode]: if self.accept('else'): self.expect('eol') - return self.codeblock() + block = self.codeblock() + return ElseNode(block) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def testcaseblock(self) -> TestCaseClauseNode: From 02ff9553dbe09f3f2b7a93221dfd28fc53926d0e Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:24:17 -0400 Subject: [PATCH 11/18] parser: add SymbolNode to preserve operators --- mesonbuild/ast/visitor.py | 6 + mesonbuild/cargo/builder.py | 20 ++- mesonbuild/cmake/interpreter.py | 14 +- mesonbuild/mparser.py | 266 ++++++++++++++++++++++++-------- mesonbuild/rewriter.py | 19 ++- 5 files changed, 239 insertions(+), 86 deletions(-) diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index a83782655..d05d3ffe5 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -55,6 +55,12 @@ class AstVisitor: def visit_BreakNode(self, node: mparser.BreakNode) -> None: self.visit_default_func(node) + def visit_SymbolNode(self, node: mparser.SymbolNode) -> None: + self.visit_default_func(node) + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + self.visit_default_func(node) + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: self.visit_default_func(node) node.args.accept(self) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 6ef66eeca..9c650b945 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -28,6 +28,10 @@ def _token(tid: str, filename: str, value: mparser.TV_TokenTypes) -> mparser.Tok return mparser.Token(tid, filename, -1, -1, -1, (-1, -1), value) +def _symbol(filename: str, val: str) -> mparser.SymbolNode: + return mparser.SymbolNode(_token('', filename, val)) + + def string(value: str, filename: str) -> mparser.StringNode: """Build A StringNode @@ -67,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode: """ args = mparser.ArgumentNode(_token('array', filename, 'unused')) args.arguments = value - return mparser.ArrayNode(args, -1, -1, -1, -1) + return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'), -1, -1, -1, -1) def identifier(value: str, filename: str) -> mparser.IdNode: @@ -97,7 +101,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, identifier(name, id_.filename), args) + return mparser.MethodNode(id_.filename, -1, -1, id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) def function(name: str, filename: str, @@ -117,7 +121,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), args) + return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: @@ -127,7 +131,7 @@ def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNod :param rhs: the right hand side of the equal :return: A compraison node """ - return mparser.ComparisonNode('==', lhs, rhs) + return mparser.ComparisonNode('==', lhs, _symbol(lhs.filename, '=='), rhs) def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: @@ -137,7 +141,7 @@ def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: :param rhs: The Right of the Node :return: The OrNode """ - return mparser.OrNode(lhs, rhs) + return mparser.OrNode(lhs, _symbol(lhs.filename, 'or'), rhs) def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: @@ -147,7 +151,7 @@ def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: :param rhs: The right of the And :return: The AndNode """ - return mparser.AndNode(lhs, rhs) + return mparser.AndNode(lhs, _symbol(lhs.filename, 'and'), rhs) def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: @@ -157,7 +161,7 @@ def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: :param filename: the string filename :return: The NotNode """ - return mparser.NotNode(_token('not', filename, ''), value) + return mparser.NotNode(_token('not', filename, ''), _symbol(filename, 'not'), value) def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.AssignmentNode: @@ -168,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), value) + return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), _symbol(filename, '='), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 0703c1b93..4f3b31e59 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -48,6 +48,7 @@ from ..mparser import ( IndexNode, MethodNode, NumberNode, + SymbolNode, ) @@ -959,6 +960,9 @@ class CMakeInterpreter: def token(tid: str = 'string', val: TYPE_mixed = '') -> Token: return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) + def symbol(val: str) -> SymbolNode: + return SymbolNode(token('', val)) + def string(value: str) -> StringNode: return StringNode(token(val=value), escape=False) @@ -984,14 +988,14 @@ class CMakeInterpreter: raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value))) def indexed(node: BaseNode, index: int) -> IndexNode: - return IndexNode(node, nodeify(index)) + return IndexNode(node, symbol('['), nodeify(index), symbol(']')) def array(elements: TYPE_mixed_list) -> ArrayNode: args = ArgumentNode(token()) if not isinstance(elements, list): elements = [args] args.arguments += [nodeify(x) for x in elements if x is not None] - return ArrayNode(args, 0, 0, 0, 0) + return ArrayNode(symbol('['), args, symbol(']'), 0, 0, 0, 0) def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: args = [] if args is None else args @@ -1002,7 +1006,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), args_n) + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), symbol('('), args_n, symbol(')')) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1014,10 +1018,10 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, id_node(name), args_n) + return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), value) + return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), symbol('='), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 6b578b5af..7f329dd0a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -26,6 +26,8 @@ if T.TYPE_CHECKING: from .ast import AstVisitor + BaseNodeT = T.TypeVar('BaseNodeT', bound='BaseNode') + # This is the regex for the supported escape sequences of a regular string # literal, like 'abc\x00' ESCAPE_SEQUENCE_SINGLE_RE = re.compile(r''' @@ -234,12 +236,14 @@ class BaseNode: end_lineno: int = field(hash=False) end_colno: int = field(hash=False) - def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: + def __init__(self, lineno: int, colno: int, filename: str, + end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: self.lineno = lineno self.colno = colno self.filename = filename self.end_lineno = end_lineno if end_lineno is not None else lineno self.end_colno = end_colno if end_colno is not None else colno + self.whitespaces = None # Attributes for the visitors self.level = 0 @@ -313,17 +317,22 @@ class ContinueNode(ElementaryNode): class BreakNode(ElementaryNode): pass +class SymbolNode(ElementaryNode[str]): + pass + @dataclass(unsafe_hash=True) class ArgumentNode(BaseNode): arguments: T.List[BaseNode] = field(hash=False) - commas: T.List[Token] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + columns: T.List[SymbolNode] = field(hash=False) kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) self.arguments = [] self.commas = [] + self.columns = [] self.kwargs = {} self.order_error = False @@ -363,20 +372,30 @@ class ArgumentNode(BaseNode): @dataclass(unsafe_hash=True) class ArrayNode(BaseNode): + lbracket: SymbolNode args: ArgumentNode + rbracket: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lbracket = lbracket self.args = args + self.rbracket = rbracket @dataclass(unsafe_hash=True) class DictNode(BaseNode): + lcurl: SymbolNode args: ArgumentNode + rcurl: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lcurl = lcurl self.args = args + self.rcurl = rcurl class EmptyNode(BaseNode): pass @@ -385,34 +404,40 @@ class EmptyNode(BaseNode): class OrNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class AndNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class ComparisonNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode ctype: COMPARISONS - def __init__(self, ctype: COMPARISONS, left: BaseNode, right: BaseNode): + def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right self.ctype = ctype @@ -423,21 +448,25 @@ class ArithmeticNode(BaseNode): right: BaseNode # TODO: use a Literal for operation operation: str + operator: SymbolNode - def __init__(self, operation: str, left: BaseNode, right: BaseNode): + def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left self.right = right self.operation = operation + self.operator = operator @dataclass(unsafe_hash=True) class NotNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, token: Token[TV_TokenTypes], value: BaseNode): + def __init__(self, token: Token[TV_TokenTypes], operator: SymbolNode, value: BaseNode): super().__init__(token.lineno, token.colno, token.filename) + self.operator = operator self.value = value @dataclass(unsafe_hash=True) @@ -447,53 +476,73 @@ class CodeBlockNode(BaseNode): def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) + self.pre_whitespaces = None self.lines = [] @dataclass(unsafe_hash=True) class IndexNode(BaseNode): iobject: BaseNode + lbracket: SymbolNode index: BaseNode + rbracket: SymbolNode - def __init__(self, iobject: BaseNode, index: BaseNode): + def __init__(self, iobject: BaseNode, lbracket: SymbolNode, index: BaseNode, rbracket: SymbolNode): super().__init__(iobject.lineno, iobject.colno, iobject.filename) self.iobject = iobject + self.lbracket = lbracket self.index = index + self.rbracket = rbracket @dataclass(unsafe_hash=True) class MethodNode(BaseNode): source_object: BaseNode + dot: SymbolNode name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, + source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename) self.source_object = source_object + self.dot = dot self.name = name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class FunctionNode(BaseNode): func_name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, + func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) self.func_name = func_name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @@ -501,46 +550,61 @@ class AssignmentNode(BaseNode): class PlusAssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): + foreach_: SymbolNode = field(hash=False) varnames: T.List[IdNode] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + column: SymbolNode = field(hash=False) items: BaseNode block: CodeBlockNode + endforeach: SymbolNode = field(hash=False) - def __init__(self, token: Token, varnames: T.List[IdNode], items: BaseNode, block: CodeBlockNode): - super().__init__(token.lineno, token.colno, token.filename) + def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], column: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode): + super().__init__(foreach_.lineno, foreach_.colno, foreach_.filename) + self.foreach_ = foreach_ self.varnames = varnames + self.commas = commas + self.column = column self.items = items self.block = block + self.endforeach = endforeach @dataclass(unsafe_hash=True) class IfNode(BaseNode): + if_: SymbolNode condition: BaseNode block: CodeBlockNode - def __init__(self, linenode: BaseNode, condition: BaseNode, block: CodeBlockNode): + def __init__(self, linenode: BaseNode, if_node: SymbolNode, condition: BaseNode, block: CodeBlockNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) + self.if_ = if_node self.condition = condition self.block = block @dataclass(unsafe_hash=True) class ElseNode(BaseNode): + else_: SymbolNode block: CodeBlockNode - def __init__(self, block: CodeBlockNode): + def __init__(self, else_: SymbolNode, block: CodeBlockNode): super().__init__(block.lineno, block.colno, block.filename) + self.else_ = else_ self.block = block @dataclass(unsafe_hash=True) @@ -548,30 +612,38 @@ class IfClauseNode(BaseNode): ifs: T.List[IfNode] = field(hash=False) elseblock: T.Union[EmptyNode, ElseNode] + endif: SymbolNode def __init__(self, linenode: BaseNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) self.ifs = [] self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename) + self.endif = None @dataclass(unsafe_hash=True) class TestCaseClauseNode(BaseNode): + testcase: SymbolNode condition: BaseNode block: CodeBlockNode + endtestcase: SymbolNode - def __init__(self, condition: BaseNode, block: CodeBlockNode): + def __init__(self, testcase: SymbolNode, condition: BaseNode, block: CodeBlockNode, endtestcase: SymbolNode): super().__init__(condition.lineno, condition.colno, condition.filename) + self.testcase = testcase self.condition = condition self.block = block + self.endtestcase = endtestcase @dataclass(unsafe_hash=True) class UMinusNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, current_location: Token, value: BaseNode): + def __init__(self, current_location: Token, operator: SymbolNode, value: BaseNode): super().__init__(current_location.lineno, current_location.colno, current_location.filename) + self.operator = operator self.value = value @@ -579,23 +651,32 @@ class UMinusNode(BaseNode): class TernaryNode(BaseNode): condition: BaseNode + questionmark: SymbolNode trueblock: BaseNode + column: SymbolNode falseblock: BaseNode - def __init__(self, condition: BaseNode, trueblock: BaseNode, falseblock: BaseNode): + def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, column: SymbolNode, falseblock: BaseNode): super().__init__(condition.lineno, condition.colno, condition.filename) self.condition = condition + self.questionmark = questionmark self.trueblock = trueblock + self.column = column self.falseblock = falseblock + @dataclass(unsafe_hash=True) class ParenthesizedNode(BaseNode): + lpar: SymbolNode = field(hash=False) inner: BaseNode + rpar: SymbolNode = field(hash=False) - def __init__(self, inner: BaseNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode, lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lpar = lpar self.inner = inner + self.rpar = rpar if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] @@ -632,13 +713,19 @@ class Parser: self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.previous = self.current + self.getsym() self.in_ternary = False + def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT: + node = node_type(*args, **kwargs) + return node + def getsym(self) -> None: self.previous = self.current try: self.current = next(self.stream) + except StopIteration: self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) @@ -683,55 +770,69 @@ class Parser: def e1(self) -> BaseNode: left = self.e2() if self.accept('plusassign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return PlusAssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(PlusAssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('assign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return AssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(AssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', self.getline(), left.lineno, left.colno) + + qm_node = self.create_node(SymbolNode, self.previous) self.in_ternary = True trueblock = self.e1() self.expect('colon') + column_node = self.create_node(SymbolNode, self.previous) falseblock = self.e1() self.in_ternary = False - return TernaryNode(left, trueblock, falseblock) + return self.create_node(TernaryNode, left, qm_node, trueblock, column_node, falseblock) return left def e2(self) -> BaseNode: left = self.e3() while self.accept('or'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid or clause.', self.getline(), left.lineno, left.colno) - left = OrNode(left, self.e3()) + left = self.create_node(OrNode, left, operator, self.e3()) return left def e3(self) -> BaseNode: left = self.e4() while self.accept('and'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid and clause.', self.getline(), left.lineno, left.colno) - left = AndNode(left, self.e4()) + left = self.create_node(AndNode, left, operator, self.e4()) return left def e4(self) -> BaseNode: left = self.e5() for nodename, operator_type in comparison_map.items(): if self.accept(nodename): - return ComparisonNode(operator_type, left, self.e5()) - if self.accept('not') and self.accept('in'): - return ComparisonNode('notin', left, self.e5()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(ComparisonNode, operator_type, left, operator, self.e5()) + if self.accept('not'): + not_token = self.previous + if self.accept('in'): + in_token = self.previous + not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1]) + not_token.value += in_token.value + operator = self.create_node(SymbolNode, not_token) + return self.create_node(ComparisonNode, 'notin', left, operator, self.e5()) return left def e5(self) -> BaseNode: @@ -746,7 +847,8 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e5muldiv()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e5muldiv()) else: break return left @@ -761,29 +863,34 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e6()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e6()) else: break return left def e6(self) -> BaseNode: if self.accept('not'): - return NotNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(NotNode, self.current, operator, self.e7()) if self.accept('dash'): - return UMinusNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(UMinusNode, self.current, operator, self.e7()) return self.e7() def e7(self) -> BaseNode: left = self.e8() block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rparen', block_start) + rpar = self.create_node(SymbolNode, self.previous) if not isinstance(left, IdNode): raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, args) + left = self.create_node(FunctionNode, left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, lpar, args, rpar) go_again = True while go_again: go_again = False @@ -798,17 +905,23 @@ class Parser: def e8(self) -> BaseNode: block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) e = self.statement() self.block_expect('rparen', block_start) - return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rpar = self.create_node(SymbolNode, self.previous) + return ParenthesizedNode(lpar, e, rpar, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lbracket'): + lbracket = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rbracket', block_start) - return ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(ArrayNode, lbracket, args, rbracket, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lcurl'): + lcurl = self.create_node(SymbolNode, block_start) key_values = self.key_values() self.block_expect('rcurl', block_start) - return DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rcurl = self.create_node(SymbolNode, self.previous) + return self.create_node(DictNode, lcurl, key_values, rcurl, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) else: return self.e9() @@ -816,34 +929,35 @@ class Parser: t = self.current if self.accept('true'): t.value = True - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('false'): t.value = False - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('id'): - return IdNode(t) + return self.create_node(IdNode, t) if self.accept('number'): - return NumberNode(t) + return self.create_node(NumberNode, t) if self.accept('string'): - return StringNode(t) + return self.create_node(StringNode, t) if self.accept('fstring'): - return FormatStringNode(t) + return self.create_node(FormatStringNode, t) if self.accept('multiline_string'): - return MultilineStringNode(t) + return self.create_node(MultilineStringNode, t) if self.accept('multiline_fstring'): - return MultilineFormatStringNode(t) + return self.create_node(MultilineFormatStringNode, t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def key_values(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) a.set_kwarg_no_check(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: raise ParseException('Only key:value pairs are valid in dict construction.', self.getline(), s.lineno, s.colno) @@ -852,20 +966,21 @@ class Parser: def args(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('comma'): - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) a.append(s) elif self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) a.set_kwarg(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: a.append(s) return a @@ -873,6 +988,7 @@ class Parser: return a def method_call(self, source_object: BaseNode) -> MethodNode: + dot = self.create_node(SymbolNode, self.previous) methodname = self.e9() if not isinstance(methodname, IdNode): if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode): @@ -882,63 +998,78 @@ class Parser: self.getline(), self.current.lineno, self.current.colno) assert isinstance(methodname.value, str) self.expect('lparen') + lpar = self.create_node(SymbolNode, self.previous) args = self.args() + rpar = self.create_node(SymbolNode, self.current) self.expect('rparen') - method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname, args) + method = self.create_node(MethodNode, methodname.filename, methodname.lineno, methodname.colno, + source_object, dot, methodname, lpar, args, rpar) if self.accept('dot'): return self.method_call(method) return method def index_call(self, source_object: BaseNode) -> IndexNode: + lbracket = self.create_node(SymbolNode, self.previous) index_statement = self.statement() self.expect('rbracket') - return IndexNode(source_object, index_statement) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(IndexNode, source_object, lbracket, index_statement, rbracket) def foreachblock(self) -> ForeachClauseNode: + foreach_ = self.create_node(SymbolNode, self.previous) self.expect('id') assert isinstance(self.previous.value, str) - varname = self.previous - varnames = [IdNode(self.previous)] + varnames = [self.create_node(IdNode, self.previous)] + commas = [] if self.accept('comma'): + commas.append(self.create_node(SymbolNode, self.previous)) self.expect('id') assert isinstance(self.previous.value, str) - varnames.append(IdNode(self.previous)) + varnames.append(self.create_node(IdNode, self.previous)) self.expect('colon') + column = self.create_node(SymbolNode, self.previous) items = self.statement() block = self.codeblock() - return ForeachClauseNode(varname, varnames, items, block) + endforeach = self.create_node(SymbolNode, self.current) + return self.create_node(ForeachClauseNode, foreach_, varnames, commas, column, items, block, endforeach) def ifblock(self) -> IfClauseNode: + if_node = self.create_node(SymbolNode, self.previous) condition = self.statement() - clause = IfClauseNode(condition) + clause = self.create_node(IfClauseNode, condition) self.expect('eol') block = self.codeblock() - clause.ifs.append(IfNode(clause, condition, block)) + clause.ifs.append(self.create_node(IfNode, clause, if_node, condition, block)) self.elseifblock(clause) clause.elseblock = self.elseblock() + clause.endif = self.create_node(SymbolNode, self.current) return clause def elseifblock(self, clause: IfClauseNode) -> None: while self.accept('elif'): + elif_ = self.create_node(SymbolNode, self.previous) s = self.statement() self.expect('eol') b = self.codeblock() - clause.ifs.append(IfNode(s, s, b)) + clause.ifs.append(self.create_node(IfNode, s, elif_, s, b)) def elseblock(self) -> T.Union[ElseNode, EmptyNode]: if self.accept('else'): + else_ = self.create_node(SymbolNode, self.previous) self.expect('eol') block = self.codeblock() - return ElseNode(block) + return ElseNode(else_, block) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def testcaseblock(self) -> TestCaseClauseNode: + testcase = self.create_node(SymbolNode, self.previous) condition = self.statement() self.expect('eol') block = self.codeblock() - return TestCaseClauseNode(condition, block) + endtestcase = SymbolNode(self.current) + return self.create_node(TestCaseClauseNode, testcase, condition, block, endtestcase) def line(self) -> BaseNode: block_start = self.current @@ -953,9 +1084,9 @@ class Parser: self.block_expect('endforeach', block_start) return forblock if self.accept('continue'): - return ContinueNode(self.current) + return self.create_node(ContinueNode, self.current) if self.accept('break'): - return BreakNode(self.current) + return self.create_node(BreakNode, self.current) if self.lexer.in_unit_test and self.accept('testcase'): block = self.testcaseblock() self.block_expect('endtestcase', block_start) @@ -963,15 +1094,20 @@ class Parser: return self.statement() def codeblock(self) -> CodeBlockNode: - block = CodeBlockNode(self.current) + block = self.create_node(CodeBlockNode, self.current) cond = True + try: while cond: curline = self.line() + if not isinstance(curline, EmptyNode): block.lines.append(curline) + cond = self.accept('eol') + except ParseException as e: e.ast = block raise + return block diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index af3210cf3..64eb2f2f1 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode import json, os, re, sys import typing as T @@ -104,6 +104,9 @@ class RequiredKeys: return wrapped +def _symbol(val: str) -> SymbolNode: + return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val)) + class MTypeBase: def __init__(self, node: T.Optional[BaseNode] = None): if node is None: @@ -189,7 +192,7 @@ class MTypeList(MTypeBase): super().__init__(node) def _new_node(self): - return ArrayNode(ArgumentNode(Token('', '', 0, 0, 0, None, '')), 0, 0, 0, 0) + return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'), 0, 0, 0, 0) def _new_element_node(self, value): # Overwrite in derived class @@ -728,7 +731,7 @@ class Rewriter: node = tgt_function.args.kwargs[extra_files_key] except StopIteration: # Target has no extra_files kwarg, create one - node = ArrayNode(ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) + node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node mark_array = False if tgt_function not in self.modified_nodes: @@ -812,17 +815,17 @@ class Rewriter: # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) + src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), src_far_node) - src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), src_fun_node) + src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) + src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), tgt_arg_node) - tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), tgt_fun_node) + tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) + tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) From 0f4891cdf46e28d900cc35d57999fecff3ba8598 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:58:32 -0400 Subject: [PATCH 12/18] parser: simplify Assignment and PlusAssignment nodes --- mesonbuild/cargo/builder.py | 2 +- mesonbuild/cmake/interpreter.py | 2 +- mesonbuild/interpreterbase/interpreterbase.py | 4 +-- mesonbuild/mparser.py | 27 +++++-------------- mesonbuild/rewriter.py | 4 +-- 5 files changed, 12 insertions(+), 27 deletions(-) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 9c650b945..76d2e8bba 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -172,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), _symbol(filename, '='), value) + return mparser.AssignmentNode(identifier(varname, filename), _symbol(filename, '='), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 4f3b31e59..88e5183bc 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -1021,7 +1021,7 @@ class CMakeInterpreter: return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), symbol('='), value) + return AssignmentNode(id_node(var_name), symbol('='), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/interpreterbase/interpreterbase.py b/mesonbuild/interpreterbase/interpreterbase.py index 5b07dc075..ea6e37ca6 100644 --- a/mesonbuild/interpreterbase/interpreterbase.py +++ b/mesonbuild/interpreterbase/interpreterbase.py @@ -192,6 +192,8 @@ class InterpreterBase: self.current_node = cur if isinstance(cur, mparser.FunctionNode): return self.function_call(cur) + elif isinstance(cur, mparser.PlusAssignmentNode): + self.evaluate_plusassign(cur) elif isinstance(cur, mparser.AssignmentNode): self.assignment(cur) elif isinstance(cur, mparser.MethodNode): @@ -229,8 +231,6 @@ class InterpreterBase: return self.evaluate_arithmeticstatement(cur) elif isinstance(cur, mparser.ForeachClauseNode): self.evaluate_foreach(cur) - elif isinstance(cur, mparser.PlusAssignmentNode): - self.evaluate_plusassign(cur) elif isinstance(cur, mparser.IndexNode): return self.evaluate_indexing(cur) elif isinstance(cur, mparser.TernaryNode): diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 7f329dd0a..ffe775037 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -530,7 +530,6 @@ class FunctionNode(BaseNode): self.args = args self.rpar = rpar - @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): @@ -538,28 +537,14 @@ class AssignmentNode(BaseNode): operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, - var_name: IdNode, operator: SymbolNode, value: BaseNode): - super().__init__(lineno, colno, filename) - self.var_name = var_name - self.operator = operator - self.value = value - - -@dataclass(unsafe_hash=True) -class PlusAssignmentNode(BaseNode): - - var_name: IdNode - operator: SymbolNode - value: BaseNode - - def __init__(self, filename: str, lineno: int, colno: int, - var_name: IdNode, operator: SymbolNode, value: BaseNode): - super().__init__(lineno, colno, filename) + def __init__(self, var_name: IdNode, operator: SymbolNode, value: BaseNode): + super().__init__(var_name.lineno, var_name.colno, var_name.filename) self.var_name = var_name self.operator = operator self.value = value +class PlusAssignmentNode(AssignmentNode): + pass @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): @@ -775,7 +760,7 @@ class Parser: if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return self.create_node(PlusAssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) + return self.create_node(PlusAssignmentNode, left, operator, value) elif self.accept('assign'): operator = self.create_node(SymbolNode, self.previous) value = self.e1() @@ -783,7 +768,7 @@ class Parser: raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return self.create_node(AssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) + return self.create_node(AssignmentNode, left, operator, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 64eb2f2f1..7527daba1 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -818,14 +818,14 @@ class Rewriter: src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) - src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) + src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) - tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) + tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id)) From 13ddf8bd025c3ff6257049cab6f3df1072ac0daf Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:41:47 -0400 Subject: [PATCH 13/18] parser: simplify by using Unary and Binary Operator Node --- mesonbuild/mparser.py | 59 +++++++++++++------------------------------ 1 file changed, 17 insertions(+), 42 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index ffe775037..00b87318c 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -401,7 +401,7 @@ class EmptyNode(BaseNode): pass @dataclass(unsafe_hash=True) -class OrNode(BaseNode): +class BinaryOperatorNode(BaseNode): left: BaseNode operator: SymbolNode @@ -413,53 +413,33 @@ class OrNode(BaseNode): self.operator = operator self.right = right -@dataclass(unsafe_hash=True) -class AndNode(BaseNode): - - left: BaseNode - operator: SymbolNode - right: BaseNode +class OrNode(BinaryOperatorNode): + pass - def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.operator = operator - self.right = right +class AndNode(BinaryOperatorNode): + pass @dataclass(unsafe_hash=True) -class ComparisonNode(BaseNode): +class ComparisonNode(BinaryOperatorNode): - left: BaseNode - operator: SymbolNode - right: BaseNode ctype: COMPARISONS def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.operator = operator - self.right = right + super().__init__(left, operator, right) self.ctype = ctype @dataclass(unsafe_hash=True) -class ArithmeticNode(BaseNode): +class ArithmeticNode(BinaryOperatorNode): - left: BaseNode - right: BaseNode # TODO: use a Literal for operation operation: str - operator: SymbolNode def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode): - super().__init__(left.lineno, left.colno, left.filename) - self.left = left - self.right = right + super().__init__(left, operator, right) self.operation = operation - self.operator = operator - @dataclass(unsafe_hash=True) -class NotNode(BaseNode): +class UnaryOperatorNode(BaseNode): operator: SymbolNode value: BaseNode @@ -469,6 +449,12 @@ class NotNode(BaseNode): self.operator = operator self.value = value +class NotNode(UnaryOperatorNode): + pass + +class UMinusNode(UnaryOperatorNode): + pass + @dataclass(unsafe_hash=True) class CodeBlockNode(BaseNode): @@ -620,18 +606,6 @@ class TestCaseClauseNode(BaseNode): self.block = block self.endtestcase = endtestcase -@dataclass(unsafe_hash=True) -class UMinusNode(BaseNode): - - operator: SymbolNode - value: BaseNode - - def __init__(self, current_location: Token, operator: SymbolNode, value: BaseNode): - super().__init__(current_location.lineno, current_location.colno, current_location.filename) - self.operator = operator - self.value = value - - @dataclass(unsafe_hash=True) class TernaryNode(BaseNode): @@ -663,6 +637,7 @@ class ParenthesizedNode(BaseNode): self.inner = inner self.rpar = rpar + if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] From 5b29eff8ad02348ff0495cd307b29259567e11df Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 10:21:53 -0400 Subject: [PATCH 14/18] parser: simplify other node constructors --- mesonbuild/cargo/builder.py | 6 +++--- mesonbuild/cmake/interpreter.py | 6 +++--- mesonbuild/mparser.py | 35 ++++++++++++++------------------- mesonbuild/rewriter.py | 10 +++++----- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 76d2e8bba..3f7f6885e 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -71,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode: """ args = mparser.ArgumentNode(_token('array', filename, 'unused')) args.arguments = value - return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'), -1, -1, -1, -1) + return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']')) def identifier(value: str, filename: str) -> mparser.IdNode: @@ -101,7 +101,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) + return mparser.MethodNode(id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) def function(name: str, filename: str, @@ -121,7 +121,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) + return mparser.FunctionNode(identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 88e5183bc..2f7cb6911 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -995,7 +995,7 @@ class CMakeInterpreter: if not isinstance(elements, list): elements = [args] args.arguments += [nodeify(x) for x in elements if x is not None] - return ArrayNode(symbol('['), args, symbol(']'), 0, 0, 0, 0) + return ArrayNode(symbol('['), args, symbol(']')) def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: args = [] if args is None else args @@ -1006,7 +1006,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), symbol('('), args_n, symbol(')')) + func_n = FunctionNode(id_node(name), symbol('('), args_n, symbol(')')) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1018,7 +1018,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) + return MethodNode(obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: return AssignmentNode(id_node(var_name), symbol('='), value) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 00b87318c..75a12aa4c 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -376,9 +376,8 @@ class ArrayNode(BaseNode): args: ArgumentNode rbracket: SymbolNode - def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode, - lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode): + super().__init__(lbracket.lineno, lbracket.colno, args.filename, end_lineno=rbracket.lineno, end_colno=rbracket.colno+1) self.lbracket = lbracket self.args = args self.rbracket = rbracket @@ -390,9 +389,8 @@ class DictNode(BaseNode): args: ArgumentNode rcurl: SymbolNode - def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode, - lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode): + super().__init__(lcurl.lineno, lcurl.colno, args.filename, end_lineno=rcurl.lineno, end_colno=rcurl.colno+1) self.lcurl = lcurl self.args = args self.rcurl = rcurl @@ -490,9 +488,8 @@ class MethodNode(BaseNode): args: ArgumentNode rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, - source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): - super().__init__(lineno, colno, filename) + def __init__(self, source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): + super().__init__(name.lineno, name.colno, name.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1) self.source_object = source_object self.dot = dot self.name = name @@ -508,9 +505,8 @@ class FunctionNode(BaseNode): args: ArgumentNode rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, - func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): - super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): + super().__init__(func_name.lineno, func_name.colno, func_name.filename, end_lineno=rpar.end_lineno, end_colno=rpar.end_colno+1) self.func_name = func_name self.lpar = lpar self.args = args @@ -631,8 +627,8 @@ class ParenthesizedNode(BaseNode): inner: BaseNode rpar: SymbolNode = field(hash=False) - def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode, lineno: int, colno: int, end_lineno: int, end_colno: int): - super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode): + super().__init__(lpar.lineno, lpar.colno, inner.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1) self.lpar = lpar self.inner = inner self.rpar = rpar @@ -850,7 +846,7 @@ class Parser: raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = self.create_node(FunctionNode, left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, lpar, args, rpar) + left = self.create_node(FunctionNode, left, lpar, args, rpar) go_again = True while go_again: go_again = False @@ -869,19 +865,19 @@ class Parser: e = self.statement() self.block_expect('rparen', block_start) rpar = self.create_node(SymbolNode, self.previous) - return ParenthesizedNode(lpar, e, rpar, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return ParenthesizedNode(lpar, e, rpar) elif self.accept('lbracket'): lbracket = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rbracket', block_start) rbracket = self.create_node(SymbolNode, self.previous) - return self.create_node(ArrayNode, lbracket, args, rbracket, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return self.create_node(ArrayNode, lbracket, args, rbracket) elif self.accept('lcurl'): lcurl = self.create_node(SymbolNode, block_start) key_values = self.key_values() self.block_expect('rcurl', block_start) rcurl = self.create_node(SymbolNode, self.previous) - return self.create_node(DictNode, lcurl, key_values, rcurl, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + return self.create_node(DictNode, lcurl, key_values, rcurl) else: return self.e9() @@ -962,8 +958,7 @@ class Parser: args = self.args() rpar = self.create_node(SymbolNode, self.current) self.expect('rparen') - method = self.create_node(MethodNode, methodname.filename, methodname.lineno, methodname.colno, - source_object, dot, methodname, lpar, args, rpar) + method = self.create_node(MethodNode, source_object, dot, methodname, lpar, args, rpar) if self.accept('dot'): return self.method_call(method) return method diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index 7527daba1..a86b97ecc 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -192,7 +192,7 @@ class MTypeList(MTypeBase): super().__init__(node) def _new_node(self): - return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'), 0, 0, 0, 0) + return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']')) def _new_element_node(self, value): # Overwrite in derived class @@ -731,7 +731,7 @@ class Rewriter: node = tgt_function.args.kwargs[extra_files_key] except StopIteration: # Target has no extra_files kwarg, create one - node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) + node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']')) tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node mark_array = False if tgt_function not in self.modified_nodes: @@ -815,16 +815,16 @@ class Rewriter: # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) + src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']')) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) + src_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) + tgt_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), From 11ef2a536c6c9fec0b048da74b36a0231ef2199a Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 10:08:24 -0400 Subject: [PATCH 15/18] parser: preserve whitespaces and comments --- mesonbuild/mparser.py | 67 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 75a12aa4c..a161842e6 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -116,7 +116,7 @@ class Lexer: self.keywords.update({'testcase', 'endtestcase'}) self.token_specification = [ # Need to be sorted longest to shortest. - ('ignore', re.compile(r'[ \t]')), + ('whitespace', re.compile(r'[ \t]+')), ('multiline_fstring', re.compile(r"f'''(.|\n)*?'''", re.M)), ('fstring', re.compile(r"f'([^'\\]|(\\.))*'")), ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), @@ -178,9 +178,7 @@ class Lexer: span_end = loc bytespan = (span_start, span_end) value = mo.group() - if tid in {'ignore', 'comment'}: - break - elif tid == 'lparen': + if tid == 'lparen': par_count += 1 elif tid == 'rparen': par_count -= 1 @@ -210,12 +208,12 @@ class Lexer: elif tid == 'eol_cont': lineno += 1 line_start = loc - break + tid = 'whitespace' elif tid == 'eol': lineno += 1 line_start = loc if par_count > 0 or bracket_count > 0 or curl_count > 0: - break + tid = 'whitespace' elif tid == 'id': if value in self.keywords: tid = value @@ -235,6 +233,7 @@ class BaseNode: filename: str = field(hash=False) end_lineno: int = field(hash=False) end_colno: int = field(hash=False) + whitespaces: T.Optional[WhitespaceNode] = field(hash=False) def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: @@ -257,6 +256,26 @@ class BaseNode: if callable(func): func(self) + def append_whitespaces(self, token: Token) -> None: + if self.whitespaces is None: + self.whitespaces = WhitespaceNode(token) + else: + self.whitespaces.append(token) + + +@dataclass(unsafe_hash=True) +class WhitespaceNode(BaseNode): + + value: str + + def __init__(self, token: Token[str]): + super().__init__(token.lineno, token.colno, token.filename) + self.value = '' + self.append(token) + + def append(self, token: Token[str]) -> None: + self.value += token.value + @dataclass(unsafe_hash=True) class ElementaryNode(T.Generic[TV_TokenTypes], BaseNode): @@ -456,6 +475,7 @@ class UMinusNode(UnaryOperatorNode): @dataclass(unsafe_hash=True) class CodeBlockNode(BaseNode): + pre_whitespaces: T.Optional[WhitespaceNode] = field(hash=False) lines: T.List[BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): @@ -463,6 +483,14 @@ class CodeBlockNode(BaseNode): self.pre_whitespaces = None self.lines = [] + def append_whitespaces(self, token: Token) -> None: + if self.lines: + self.lines[-1].append_whitespaces(token) + elif self.pre_whitespaces is None: + self.pre_whitespaces = WhitespaceNode(token) + else: + self.pre_whitespaces.append(token) + @dataclass(unsafe_hash=True) class IndexNode(BaseNode): @@ -669,12 +697,16 @@ class Parser: self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.previous = self.current + self.current_ws: T.List[Token] = [] self.getsym() self.in_ternary = False def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT: node = node_type(*args, **kwargs) + for ws_token in self.current_ws: + node.append_whitespaces(ws_token) + self.current_ws = [] return node def getsym(self) -> None: @@ -682,6 +714,12 @@ class Parser: try: self.current = next(self.stream) + while self.current.tid in {'eol', 'comment', 'whitespace'}: + self.current_ws.append(self.current) + if self.current.tid == 'eol': + break + self.current = next(self.stream) + except StopIteration: self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) @@ -782,11 +820,17 @@ class Parser: operator = self.create_node(SymbolNode, self.previous) return self.create_node(ComparisonNode, operator_type, left, operator, self.e5()) if self.accept('not'): + ws = self.current_ws.copy() not_token = self.previous if self.accept('in'): in_token = self.previous + self.current_ws = self.current_ws[len(ws):] # remove whitespaces between not and in + temp_node = EmptyNode(in_token.lineno, in_token.colno, in_token.filename) + for w in ws: + temp_node.append_whitespaces(w) + not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1]) - not_token.value += in_token.value + not_token.value += temp_node.whitespaces.value + in_token.value operator = self.create_node(SymbolNode, not_token) return self.create_node(ComparisonNode, 'notin', left, operator, self.e5()) return left @@ -1054,6 +1098,10 @@ class Parser: try: while cond: + for ws_token in self.current_ws: + block.append_whitespaces(ws_token) + self.current_ws = [] + curline = self.line() if not isinstance(curline, EmptyNode): @@ -1065,4 +1113,9 @@ class Parser: e.ast = block raise + # Remaining whitespaces will not be catched since there are no more nodes + for ws_token in self.current_ws: + block.append_whitespaces(ws_token) + self.current_ws = [] + return block From 6a18ae48b33c345185a8eda49e93adb1fb4594a9 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Mon, 28 Aug 2023 11:42:55 -0400 Subject: [PATCH 16/18] ast: fully resolve nodes for add operation Otherwise, string + stringmethod results in a list of two strings instead of the concatenation of the strings --- mesonbuild/ast/interpreter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py index da2119c89..c51af09f4 100644 --- a/mesonbuild/ast/interpreter.py +++ b/mesonbuild/ast/interpreter.py @@ -371,8 +371,8 @@ class AstInterpreter(InterpreterBase): elif isinstance(node, ArithmeticNode): if node.operation != 'add': return None # Only handle string and array concats - l = quick_resolve(node.left) - r = quick_resolve(node.right) + l = self.resolve_node(node.left, include_unknown_args, id_loop_detect) + r = self.resolve_node(node.right, include_unknown_args, id_loop_detect) if isinstance(l, str) and isinstance(r, str): result = l + r # String concatenation detected else: From d3a26d158e4607ce677404920ad72508eb6f9de2 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 12:46:18 -0400 Subject: [PATCH 17/18] raw printer this printer preserves all whitespaces and comments in original meson.build file. It will be useful for rewrite and potential auto-formatter --- docs/markdown/Syntax.md | 2 +- mesonbuild/ast/printer.py | 219 ++++++++++++++++++++++++ run_format_tests.py | 1 + test cases/unit/118 rewrite/meson.build | 189 ++++++++++++++++++++ unittests/rewritetests.py | 22 +++ 5 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 test cases/unit/118 rewrite/meson.build diff --git a/docs/markdown/Syntax.md b/docs/markdown/Syntax.md index b69ad00ae..59ec5f7ba 100644 --- a/docs/markdown/Syntax.md +++ b/docs/markdown/Syntax.md @@ -109,7 +109,7 @@ Strings in Meson are declared with single quotes. To enter a literal single quote do it like this: ```meson -single quote = 'contains a \' character' +single_quote = 'contains a \' character' ``` The full list of escape sequences is: diff --git a/mesonbuild/ast/printer.py b/mesonbuild/ast/printer.py index 410aabd46..155b5fc5e 100644 --- a/mesonbuild/ast/printer.py +++ b/mesonbuild/ast/printer.py @@ -18,6 +18,8 @@ from __future__ import annotations from .. import mparser from .visitor import AstVisitor + +from itertools import zip_longest import re import typing as T @@ -248,6 +250,223 @@ class AstPrinter(AstVisitor): else: self.result = re.sub(r', $', '', self.result) +class RawPrinter(AstVisitor): + + def __init__(self): + self.result = '' + + def visit_default_func(self, node: mparser.BaseNode): + self.result += node.value + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_unary_operator(self, node: mparser.UnaryOperatorNode): + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_binary_operator(self, node: mparser.BinaryOperatorNode): + node.left.accept(self) + node.operator.accept(self) + node.right.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_BooleanNode(self, node: mparser.BooleanNode) -> None: + self.result += 'true' if node.value else 'false' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_NumberNode(self, node: mparser.NumberNode) -> None: + self.result += node.raw_value + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_StringNode(self, node: mparser.StringNode) -> None: + self.result += f"'{node.raw_value}'" + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None: + self.result += f"'''{node.value}'''" + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: + self.result += 'f' + self.visit_StringNode(node) + + def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None: + self.result += 'f' + self.visit_MultilineStringNode(node) + + def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: + self.result += 'continue' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_BreakNode(self, node: mparser.BreakNode) -> None: + self.result += 'break' + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: + node.lbracket.accept(self) + node.args.accept(self) + node.rbracket.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_DictNode(self, node: mparser.DictNode) -> None: + node.lcurl.accept(self) + node.args.accept(self) + node.rcurl.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None: + node.lpar.accept(self) + node.inner.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_OrNode(self, node: mparser.OrNode) -> None: + self.visit_binary_operator(node) + + def visit_AndNode(self, node: mparser.AndNode) -> None: + self.visit_binary_operator(node) + + def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None: + self.visit_binary_operator(node) + + def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None: + self.visit_binary_operator(node) + + def visit_NotNode(self, node: mparser.NotNode) -> None: + self.visit_unary_operator(node) + + def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None: + if node.pre_whitespaces: + node.pre_whitespaces.accept(self) + for i in node.lines: + i.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_IndexNode(self, node: mparser.IndexNode) -> None: + node.iobject.accept(self) + node.lbracket.accept(self) + node.index.accept(self) + node.rbracket.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_MethodNode(self, node: mparser.MethodNode) -> None: + node.source_object.accept(self) + node.dot.accept(self) + node.name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: + node.func_name.accept(self) + node.lpar.accept(self) + node.args.accept(self) + node.rpar.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: + node.var_name.accept(self) + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: + node.var_name.accept(self) + node.operator.accept(self) + node.value.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: + node.foreach_.accept(self) + for varname, comma in zip_longest(node.varnames, node.commas): + varname.accept(self) + if comma is not None: + comma.accept(self) + node.column.accept(self) + node.items.accept(self) + node.block.accept(self) + node.endforeach.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: + for i in node.ifs: + i.accept(self) + if not isinstance(node.elseblock, mparser.EmptyNode): + node.elseblock.accept(self) + node.endif.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_UMinusNode(self, node: mparser.UMinusNode) -> None: + self.visit_unary_operator(node) + + def visit_IfNode(self, node: mparser.IfNode) -> None: + node.if_.accept(self) + node.condition.accept(self) + node.block.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ElseNode(self, node: mparser.ElseNode) -> None: + node.else_.accept(self) + node.block.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: + node.condition.accept(self) + node.questionmark.accept(self) + node.trueblock.accept(self) + node.column.accept(self) + node.falseblock.accept(self) + if node.whitespaces: + node.whitespaces.accept(self) + + def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None: + commas_iter = iter(node.commas) + + for arg in node.arguments: + arg.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + assert len(node.columns) == len(node.kwargs) + for (key, val), column in zip(node.kwargs.items(), node.columns): + key.accept(self) + column.accept(self) + val.accept(self) + try: + comma = next(commas_iter) + comma.accept(self) + except StopIteration: + pass + + if node.whitespaces: + node.whitespaces.accept(self) + class AstJSONPrinter(AstVisitor): def __init__(self) -> None: self.result: T.Dict[str, T.Any] = {} diff --git a/run_format_tests.py b/run_format_tests.py index 1f41f3d10..1bf997ccc 100644 --- a/run_format_tests.py +++ b/run_format_tests.py @@ -63,6 +63,7 @@ def check_format() -> None: 'work area', '.eggs', '_cache', # e.g. .mypy_cache 'venv', # virtualenvs have DOS line endings + '118 rewrite', # we explicitly test for tab in meson.build file } for (root, _, filenames) in os.walk('.'): if any([x in root for x in skip_dirs]): diff --git a/test cases/unit/118 rewrite/meson.build b/test cases/unit/118 rewrite/meson.build new file mode 100644 index 000000000..cb8c15fe6 --- /dev/null +++ b/test cases/unit/118 rewrite/meson.build @@ -0,0 +1,189 @@ +# This file should expose all possible meson syntaxes + # and ensure the AstInterpreter and RawPrinter are able + + # to parse and write a file identical to the original. + + project ( # project comment 1 + # project comment 2 + 'rewrite' , # argument comment + # project comment 3 + 'cpp', + 'c', + default_options: [ + 'unity=on', + 'unity_size=50', # number of cpp / unity. default is 4... + 'warning_level=2', # eqv to /W3 + 'werror=true', # treat warnings as errors + 'b_ndebug=if-release', # disable assert in Release + 'cpp_eh=a', # /EHa exception handling + 'cpp_std=c++17', + 'cpp_winlibs=' + ','.join([ # array comment + # in array + # comment + 'kernel32.lib', + 'user32.lib', + 'gdi32.lib', + 'winspool.lib', + 'comdlg32.lib', + 'advapi32.lib', + 'shell32.lib' + # before comma comment + , + # after comma comment + 'ole32.lib', + 'oleaut32.lib', + 'uuid.lib', + 'odbc32.lib', + 'odbccp32.lib', + 'Delayimp.lib', # For delay loaded dll + 'OLDNAMES.lib', + 'dbghelp.lib', + 'psapi.lib', + ]), + ], + meson_version: '>=1.2', + version: '1.0.0', + ) # project comment 4 + +cppcoro_dep = dependency('andreasbuhr-cppcoro-cppcoro') +cppcoro = declare_dependency( + dependencies: [cppcoro_dep.partial_dependency( + includes: true, + link_args: true, + links: true, + sources: true, + )], + # '/await:strict' allows to use rather than with C++17. + # We can remove '/await:strict' once we update to C++20. + compile_args: ['/await:strict'], + # includes:true doesn't work for now in partial_dependency() + # This line could be removed once https://github.com/mesonbuild/meson/pull/10122 is released. + include_directories: cppcoro_dep.get_variable('includedir1'), +) + + +if get_option('unicode') #if comment +#if comment 2 + mfc=cpp_compiler.find_library(get_option('debug')?'mfc140ud':'mfc140u') + # if comment 3 +else#elsecommentnowhitespaces + # else comment 1 + mfc = cpp_compiler.find_library( get_option( 'debug' ) ? 'mfc140d' : 'mfc140') +# else comment 2 +endif #endif comment + + +assert(1 in [1, 2], '''1 should be in [1, 2]''') +assert(3 not in [1, 2], '''3 shouldn't be in [1, 2]''') +assert(not (3 in [1, 2]), '''3 shouldn't be in [1, 2]''') + +assert('b' in ['a', 'b'], ''''b' should be in ['a', 'b']''') +assert('c' not in ['a', 'b'], ''''c' shouldn't be in ['a', 'b']''') + +assert(exe1 in [exe1, exe2], ''''exe1 should be in [exe1, exe2]''') +assert(exe3 not in [exe1, exe2], ''''exe3 shouldn't be in [exe1, exe2]''') + +assert('a' in {'a': 'b'}, '''1 should be in {'a': 'b'}''') +assert('b'not in{'a':'b'}, '''1 should be in {'a': 'b'}''') + +assert('a'in'abc') +assert('b' not in 'def') + + +w = 'world' +d = {'a': 1, 'b': 0b10101010, 'c': 'pi', 'd': '''a +b +c''', 'e': f'hello @w@', 'f': f'''triple + formatted + string # this is not a comment + hello @w@ +''', 'g': [1, 2, 3], + + 'h' # comment a + : # comment b +0xDEADBEEF # comment c +, # comment d +'hh': 0xfeedc0de, # lowercase hexa +'hhh': 0XaBcD0123, # mixed case hexa +'oo': 0O123456, # upper O octa +'bb': 0B1111, # upper B binary +'i': {'aa': 11, # this is a comment + 'bb': 22}, # a comment inside a dict +'o': 0o754, +'m': -12, # minus number +'eq': 1 + 3 - 3 % 4 + -( 7 * 8 ), +} # end of dict comment + +hw = d['e'] +one = d['g'][0] + w += '!' + + +components = { + 'foo': ['foo.c'], + 'bar': ['bar.c'], + 'baz': ['baz.c'], # this line is indented with a tab! +} + +# compute a configuration based on system dependencies, custom logic +conf = configuration_data() +conf.set('USE_FOO', 1) + +# Determine the sources to compile +sources_to_compile = [] +foreach name, sources : components + if conf.get('USE_@0@'.format(name.to_upper()), 0) == 1 + sources_to_compile += sources + endif +endforeach + + +items = ['a', 'continue', 'b', 'break', 'c'] +result = [] +foreach i : items + if i == 'continue' + continue + elif i == 'break' + break + endif + result += i +endforeach +# result is ['a', 'b'] + + + +if a and b + # do something +endif +if c or d + # do something +endif +if not e + # do something +endif +if not (f or g) + # do something +endif + +single_quote = 'contains a \' character' +string_escapes = '\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}' +no_string_escapes = '''\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}''' + +# FIXME: is it supposed to work? (cont_eol inside string) +# cont_string = 'blablabla\ +# blablabla' + +# FIXME: is it supposed to work? (cont_eol with whitespace and comments after) +# if a \ # comment in cont 1 +# and b \ # comment in cont 2 +# or c # comment in cont 3 +# message('ok') +# endif + +if a \ + or b + debug('help!') +endif + + +# End of file comment with no linebreak \ No newline at end of file diff --git a/unittests/rewritetests.py b/unittests/rewritetests.py index ca30fe9c2..c33884442 100644 --- a/unittests/rewritetests.py +++ b/unittests/rewritetests.py @@ -13,11 +13,15 @@ # limitations under the License. import subprocess +from itertools import zip_longest import json import os +from pathlib import Path import shutil import unittest +from mesonbuild.ast import IntrospectionInterpreter, AstIDGenerator +from mesonbuild.ast.printer import RawPrinter from mesonbuild.mesonlib import windows_proof_rmtree from .baseplatformtests import BasePlatformTests @@ -396,3 +400,21 @@ class RewriterTests(BasePlatformTests): # Check the written file out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) self.assertDictEqual(out, expected) + + def test_raw_printer_is_idempotent(self): + test_path = Path(self.unit_test_dir, '118 rewrite') + meson_build_file = test_path / 'meson.build' + # original_contents = meson_build_file.read_bytes() + original_contents = meson_build_file.read_text(encoding='utf-8') + + interpreter = IntrospectionInterpreter(test_path, '', 'ninja', visitors = [AstIDGenerator()]) + interpreter.analyze() + + printer = RawPrinter() + interpreter.ast.accept(printer) + # new_contents = printer.result.encode('utf-8') + new_contents = printer.result + + # Do it line per line because it is easier to debug like that + for orig_line, new_line in zip_longest(original_contents.splitlines(), new_contents.splitlines()): + self.assertEqual(orig_line, new_line) From 14e35b63c02a16f69dd1ad5bde775e6868965eb2 Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Wed, 23 Aug 2023 15:39:43 -0400 Subject: [PATCH 18/18] parser: allow whitespaces and comments in cont_eol FIXME: another approach would be to consider cont_eol as comment (i.e. add backslash and whitespaces to the comment regex). In both cases it works until we want to parse comments separately. TODO?: handle eol_cont inside a string (to split long string without breaking lines). Probably a bad idea and better to simply join a multiline string. --- mesonbuild/mparser.py | 2 +- test cases/unit/118 rewrite/meson.build | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index a161842e6..0f63c9e5a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -121,7 +121,7 @@ class Lexer: ('fstring', re.compile(r"f'([^'\\]|(\\.))*'")), ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), ('number', re.compile(r'0[bB][01]+|0[oO][0-7]+|0[xX][0-9a-fA-F]+|0|[1-9]\d*')), - ('eol_cont', re.compile(r'\\\n')), + ('eol_cont', re.compile(r'\\[ \t]*(#.*)?\n')), ('eol', re.compile(r'\n')), ('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)), ('comment', re.compile(r'#.*')), diff --git a/test cases/unit/118 rewrite/meson.build b/test cases/unit/118 rewrite/meson.build index cb8c15fe6..7d0330b9e 100644 --- a/test cases/unit/118 rewrite/meson.build +++ b/test cases/unit/118 rewrite/meson.build @@ -173,12 +173,12 @@ no_string_escapes = '''\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITA # cont_string = 'blablabla\ # blablabla' -# FIXME: is it supposed to work? (cont_eol with whitespace and comments after) -# if a \ # comment in cont 1 -# and b \ # comment in cont 2 -# or c # comment in cont 3 -# message('ok') -# endif +# cont_eol with whitespace and comments after +if a \ # comment in cont 1 + and b \ # comment in cont 2 + or c # comment in cont 3 + message('ok') +endif if a \ or b