From 02ff9553dbe09f3f2b7a93221dfd28fc53926d0e Mon Sep 17 00:00:00 2001 From: Charles Brunet Date: Fri, 25 Aug 2023 09:24:17 -0400 Subject: [PATCH] parser: add SymbolNode to preserve operators --- mesonbuild/ast/visitor.py | 6 + mesonbuild/cargo/builder.py | 20 ++- mesonbuild/cmake/interpreter.py | 14 +- mesonbuild/mparser.py | 266 ++++++++++++++++++++++++-------- mesonbuild/rewriter.py | 19 ++- 5 files changed, 239 insertions(+), 86 deletions(-) diff --git a/mesonbuild/ast/visitor.py b/mesonbuild/ast/visitor.py index a83782655..d05d3ffe5 100644 --- a/mesonbuild/ast/visitor.py +++ b/mesonbuild/ast/visitor.py @@ -55,6 +55,12 @@ class AstVisitor: def visit_BreakNode(self, node: mparser.BreakNode) -> None: self.visit_default_func(node) + def visit_SymbolNode(self, node: mparser.SymbolNode) -> None: + self.visit_default_func(node) + + def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None: + self.visit_default_func(node) + def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: self.visit_default_func(node) node.args.accept(self) diff --git a/mesonbuild/cargo/builder.py b/mesonbuild/cargo/builder.py index 6ef66eeca..9c650b945 100644 --- a/mesonbuild/cargo/builder.py +++ b/mesonbuild/cargo/builder.py @@ -28,6 +28,10 @@ def _token(tid: str, filename: str, value: mparser.TV_TokenTypes) -> mparser.Tok return mparser.Token(tid, filename, -1, -1, -1, (-1, -1), value) +def _symbol(filename: str, val: str) -> mparser.SymbolNode: + return mparser.SymbolNode(_token('', filename, val)) + + def string(value: str, filename: str) -> mparser.StringNode: """Build A StringNode @@ -67,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode: """ args = mparser.ArgumentNode(_token('array', filename, 'unused')) args.arguments = value - return mparser.ArrayNode(args, -1, -1, -1, -1) + return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'), -1, -1, -1, -1) def identifier(value: str, filename: str) -> mparser.IdNode: @@ -97,7 +101,7 @@ def method(name: str, id_: mparser.IdNode, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} - return mparser.MethodNode(id_.filename, -1, -1, id_, identifier(name, id_.filename), args) + return mparser.MethodNode(id_.filename, -1, -1, id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')')) def function(name: str, filename: str, @@ -117,7 +121,7 @@ def function(name: str, filename: str, args.arguments = pos if kw is not None: args.kwargs = {identifier(k, filename): v for k, v in kw.items()} - return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), args) + return mparser.FunctionNode(filename, -1, -1, -1, -1, identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')')) def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: @@ -127,7 +131,7 @@ def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNod :param rhs: the right hand side of the equal :return: A compraison node """ - return mparser.ComparisonNode('==', lhs, rhs) + return mparser.ComparisonNode('==', lhs, _symbol(lhs.filename, '=='), rhs) def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: @@ -137,7 +141,7 @@ def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: :param rhs: The Right of the Node :return: The OrNode """ - return mparser.OrNode(lhs, rhs) + return mparser.OrNode(lhs, _symbol(lhs.filename, 'or'), rhs) def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: @@ -147,7 +151,7 @@ def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: :param rhs: The right of the And :return: The AndNode """ - return mparser.AndNode(lhs, rhs) + return mparser.AndNode(lhs, _symbol(lhs.filename, 'and'), rhs) def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: @@ -157,7 +161,7 @@ def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: :param filename: the string filename :return: The NotNode """ - return mparser.NotNode(_token('not', filename, ''), value) + return mparser.NotNode(_token('not', filename, ''), _symbol(filename, 'not'), value) def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.AssignmentNode: @@ -168,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi :param filename: The filename :return: An AssignmentNode """ - return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), value) + return mparser.AssignmentNode(filename, -1, -1, identifier(varname, filename), _symbol(filename, '='), value) def block(filename: str) -> mparser.CodeBlockNode: diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py index 0703c1b93..4f3b31e59 100644 --- a/mesonbuild/cmake/interpreter.py +++ b/mesonbuild/cmake/interpreter.py @@ -48,6 +48,7 @@ from ..mparser import ( IndexNode, MethodNode, NumberNode, + SymbolNode, ) @@ -959,6 +960,9 @@ class CMakeInterpreter: def token(tid: str = 'string', val: TYPE_mixed = '') -> Token: return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) + def symbol(val: str) -> SymbolNode: + return SymbolNode(token('', val)) + def string(value: str) -> StringNode: return StringNode(token(val=value), escape=False) @@ -984,14 +988,14 @@ class CMakeInterpreter: raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value))) def indexed(node: BaseNode, index: int) -> IndexNode: - return IndexNode(node, nodeify(index)) + return IndexNode(node, symbol('['), nodeify(index), symbol(']')) def array(elements: TYPE_mixed_list) -> ArrayNode: args = ArgumentNode(token()) if not isinstance(elements, list): elements = [args] args.arguments += [nodeify(x) for x in elements if x is not None] - return ArrayNode(args, 0, 0, 0, 0) + return ArrayNode(symbol('['), args, symbol(']'), 0, 0, 0, 0) def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: args = [] if args is None else args @@ -1002,7 +1006,7 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), args_n) + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, id_node(name), symbol('('), args_n, symbol(')')) return func_n def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: @@ -1014,10 +1018,10 @@ class CMakeInterpreter: args = [args] args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} - return MethodNode(self.subdir.as_posix(), 0, 0, obj, id_node(name), args_n) + return MethodNode(self.subdir.as_posix(), 0, 0, obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')')) def assign(var_name: str, value: BaseNode) -> AssignmentNode: - return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), value) + return AssignmentNode(self.subdir.as_posix(), 0, 0, id_node(var_name), symbol('='), value) # Generate the root code block and the project function call root_cb = CodeBlockNode(token()) diff --git a/mesonbuild/mparser.py b/mesonbuild/mparser.py index 6b578b5af..7f329dd0a 100644 --- a/mesonbuild/mparser.py +++ b/mesonbuild/mparser.py @@ -26,6 +26,8 @@ if T.TYPE_CHECKING: from .ast import AstVisitor + BaseNodeT = T.TypeVar('BaseNodeT', bound='BaseNode') + # This is the regex for the supported escape sequences of a regular string # literal, like 'abc\x00' ESCAPE_SEQUENCE_SINGLE_RE = re.compile(r''' @@ -234,12 +236,14 @@ class BaseNode: end_lineno: int = field(hash=False) end_colno: int = field(hash=False) - def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: + def __init__(self, lineno: int, colno: int, filename: str, + end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: self.lineno = lineno self.colno = colno self.filename = filename self.end_lineno = end_lineno if end_lineno is not None else lineno self.end_colno = end_colno if end_colno is not None else colno + self.whitespaces = None # Attributes for the visitors self.level = 0 @@ -313,17 +317,22 @@ class ContinueNode(ElementaryNode): class BreakNode(ElementaryNode): pass +class SymbolNode(ElementaryNode[str]): + pass + @dataclass(unsafe_hash=True) class ArgumentNode(BaseNode): arguments: T.List[BaseNode] = field(hash=False) - commas: T.List[Token] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + columns: T.List[SymbolNode] = field(hash=False) kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False) def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) self.arguments = [] self.commas = [] + self.columns = [] self.kwargs = {} self.order_error = False @@ -363,20 +372,30 @@ class ArgumentNode(BaseNode): @dataclass(unsafe_hash=True) class ArrayNode(BaseNode): + lbracket: SymbolNode args: ArgumentNode + rbracket: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lbracket = lbracket self.args = args + self.rbracket = rbracket @dataclass(unsafe_hash=True) class DictNode(BaseNode): + lcurl: SymbolNode args: ArgumentNode + rcurl: SymbolNode - def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode, + lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lcurl = lcurl self.args = args + self.rcurl = rcurl class EmptyNode(BaseNode): pass @@ -385,34 +404,40 @@ class EmptyNode(BaseNode): class OrNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class AndNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode - def __init__(self, left: BaseNode, right: BaseNode): + def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right @dataclass(unsafe_hash=True) class ComparisonNode(BaseNode): left: BaseNode + operator: SymbolNode right: BaseNode ctype: COMPARISONS - def __init__(self, ctype: COMPARISONS, left: BaseNode, right: BaseNode): + def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left + self.operator = operator self.right = right self.ctype = ctype @@ -423,21 +448,25 @@ class ArithmeticNode(BaseNode): right: BaseNode # TODO: use a Literal for operation operation: str + operator: SymbolNode - def __init__(self, operation: str, left: BaseNode, right: BaseNode): + def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode): super().__init__(left.lineno, left.colno, left.filename) self.left = left self.right = right self.operation = operation + self.operator = operator @dataclass(unsafe_hash=True) class NotNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, token: Token[TV_TokenTypes], value: BaseNode): + def __init__(self, token: Token[TV_TokenTypes], operator: SymbolNode, value: BaseNode): super().__init__(token.lineno, token.colno, token.filename) + self.operator = operator self.value = value @dataclass(unsafe_hash=True) @@ -447,53 +476,73 @@ class CodeBlockNode(BaseNode): def __init__(self, token: Token[TV_TokenTypes]): super().__init__(token.lineno, token.colno, token.filename) + self.pre_whitespaces = None self.lines = [] @dataclass(unsafe_hash=True) class IndexNode(BaseNode): iobject: BaseNode + lbracket: SymbolNode index: BaseNode + rbracket: SymbolNode - def __init__(self, iobject: BaseNode, index: BaseNode): + def __init__(self, iobject: BaseNode, lbracket: SymbolNode, index: BaseNode, rbracket: SymbolNode): super().__init__(iobject.lineno, iobject.colno, iobject.filename) self.iobject = iobject + self.lbracket = lbracket self.index = index + self.rbracket = rbracket @dataclass(unsafe_hash=True) class MethodNode(BaseNode): source_object: BaseNode + dot: SymbolNode name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, + source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename) self.source_object = source_object + self.dot = dot self.name = name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class FunctionNode(BaseNode): func_name: IdNode + lpar: SymbolNode args: ArgumentNode + rpar: SymbolNode - def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: IdNode, args: ArgumentNode): + def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, + func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode): super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) self.func_name = func_name + self.lpar = lpar self.args = args + self.rpar = rpar @dataclass(unsafe_hash=True) class AssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @@ -501,46 +550,61 @@ class AssignmentNode(BaseNode): class PlusAssignmentNode(BaseNode): var_name: IdNode + operator: SymbolNode value: BaseNode - def __init__(self, filename: str, lineno: int, colno: int, var_name: IdNode, value: BaseNode): + def __init__(self, filename: str, lineno: int, colno: int, + var_name: IdNode, operator: SymbolNode, value: BaseNode): super().__init__(lineno, colno, filename) self.var_name = var_name + self.operator = operator self.value = value @dataclass(unsafe_hash=True) class ForeachClauseNode(BaseNode): + foreach_: SymbolNode = field(hash=False) varnames: T.List[IdNode] = field(hash=False) + commas: T.List[SymbolNode] = field(hash=False) + column: SymbolNode = field(hash=False) items: BaseNode block: CodeBlockNode + endforeach: SymbolNode = field(hash=False) - def __init__(self, token: Token, varnames: T.List[IdNode], items: BaseNode, block: CodeBlockNode): - super().__init__(token.lineno, token.colno, token.filename) + def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], column: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode): + super().__init__(foreach_.lineno, foreach_.colno, foreach_.filename) + self.foreach_ = foreach_ self.varnames = varnames + self.commas = commas + self.column = column self.items = items self.block = block + self.endforeach = endforeach @dataclass(unsafe_hash=True) class IfNode(BaseNode): + if_: SymbolNode condition: BaseNode block: CodeBlockNode - def __init__(self, linenode: BaseNode, condition: BaseNode, block: CodeBlockNode): + def __init__(self, linenode: BaseNode, if_node: SymbolNode, condition: BaseNode, block: CodeBlockNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) + self.if_ = if_node self.condition = condition self.block = block @dataclass(unsafe_hash=True) class ElseNode(BaseNode): + else_: SymbolNode block: CodeBlockNode - def __init__(self, block: CodeBlockNode): + def __init__(self, else_: SymbolNode, block: CodeBlockNode): super().__init__(block.lineno, block.colno, block.filename) + self.else_ = else_ self.block = block @dataclass(unsafe_hash=True) @@ -548,30 +612,38 @@ class IfClauseNode(BaseNode): ifs: T.List[IfNode] = field(hash=False) elseblock: T.Union[EmptyNode, ElseNode] + endif: SymbolNode def __init__(self, linenode: BaseNode): super().__init__(linenode.lineno, linenode.colno, linenode.filename) self.ifs = [] self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename) + self.endif = None @dataclass(unsafe_hash=True) class TestCaseClauseNode(BaseNode): + testcase: SymbolNode condition: BaseNode block: CodeBlockNode + endtestcase: SymbolNode - def __init__(self, condition: BaseNode, block: CodeBlockNode): + def __init__(self, testcase: SymbolNode, condition: BaseNode, block: CodeBlockNode, endtestcase: SymbolNode): super().__init__(condition.lineno, condition.colno, condition.filename) + self.testcase = testcase self.condition = condition self.block = block + self.endtestcase = endtestcase @dataclass(unsafe_hash=True) class UMinusNode(BaseNode): + operator: SymbolNode value: BaseNode - def __init__(self, current_location: Token, value: BaseNode): + def __init__(self, current_location: Token, operator: SymbolNode, value: BaseNode): super().__init__(current_location.lineno, current_location.colno, current_location.filename) + self.operator = operator self.value = value @@ -579,23 +651,32 @@ class UMinusNode(BaseNode): class TernaryNode(BaseNode): condition: BaseNode + questionmark: SymbolNode trueblock: BaseNode + column: SymbolNode falseblock: BaseNode - def __init__(self, condition: BaseNode, trueblock: BaseNode, falseblock: BaseNode): + def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, column: SymbolNode, falseblock: BaseNode): super().__init__(condition.lineno, condition.colno, condition.filename) self.condition = condition + self.questionmark = questionmark self.trueblock = trueblock + self.column = column self.falseblock = falseblock + @dataclass(unsafe_hash=True) class ParenthesizedNode(BaseNode): + lpar: SymbolNode = field(hash=False) inner: BaseNode + rpar: SymbolNode = field(hash=False) - def __init__(self, inner: BaseNode, lineno: int, colno: int, end_lineno: int, end_colno: int): + def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode, lineno: int, colno: int, end_lineno: int, end_colno: int): super().__init__(lineno, colno, inner.filename, end_lineno=end_lineno, end_colno=end_colno) + self.lpar = lpar self.inner = inner + self.rpar = rpar if T.TYPE_CHECKING: COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] @@ -632,13 +713,19 @@ class Parser: self.stream = self.lexer.lex(filename) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.previous = self.current + self.getsym() self.in_ternary = False + def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT: + node = node_type(*args, **kwargs) + return node + def getsym(self) -> None: self.previous = self.current try: self.current = next(self.stream) + except StopIteration: self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) @@ -683,55 +770,69 @@ class Parser: def e1(self) -> BaseNode: left = self.e2() if self.accept('plusassign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return PlusAssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(PlusAssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('assign'): + operator = self.create_node(SymbolNode, self.previous) value = self.e1() if not isinstance(left, IdNode): raise ParseException('Assignment target must be an id.', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - return AssignmentNode(left.filename, left.lineno, left.colno, left, value) + return self.create_node(AssignmentNode, left.filename, left.lineno, left.colno, left, operator, value) elif self.accept('questionmark'): if self.in_ternary: raise ParseException('Nested ternary operators are not allowed.', self.getline(), left.lineno, left.colno) + + qm_node = self.create_node(SymbolNode, self.previous) self.in_ternary = True trueblock = self.e1() self.expect('colon') + column_node = self.create_node(SymbolNode, self.previous) falseblock = self.e1() self.in_ternary = False - return TernaryNode(left, trueblock, falseblock) + return self.create_node(TernaryNode, left, qm_node, trueblock, column_node, falseblock) return left def e2(self) -> BaseNode: left = self.e3() while self.accept('or'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid or clause.', self.getline(), left.lineno, left.colno) - left = OrNode(left, self.e3()) + left = self.create_node(OrNode, left, operator, self.e3()) return left def e3(self) -> BaseNode: left = self.e4() while self.accept('and'): + operator = self.create_node(SymbolNode, self.previous) if isinstance(left, EmptyNode): raise ParseException('Invalid and clause.', self.getline(), left.lineno, left.colno) - left = AndNode(left, self.e4()) + left = self.create_node(AndNode, left, operator, self.e4()) return left def e4(self) -> BaseNode: left = self.e5() for nodename, operator_type in comparison_map.items(): if self.accept(nodename): - return ComparisonNode(operator_type, left, self.e5()) - if self.accept('not') and self.accept('in'): - return ComparisonNode('notin', left, self.e5()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(ComparisonNode, operator_type, left, operator, self.e5()) + if self.accept('not'): + not_token = self.previous + if self.accept('in'): + in_token = self.previous + not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1]) + not_token.value += in_token.value + operator = self.create_node(SymbolNode, not_token) + return self.create_node(ComparisonNode, 'notin', left, operator, self.e5()) return left def e5(self) -> BaseNode: @@ -746,7 +847,8 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e5muldiv()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e5muldiv()) else: break return left @@ -761,29 +863,34 @@ class Parser: while True: op = self.accept_any(tuple(op_map.keys())) if op: - left = ArithmeticNode(op_map[op], left, self.e6()) + operator = self.create_node(SymbolNode, self.previous) + left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e6()) else: break return left def e6(self) -> BaseNode: if self.accept('not'): - return NotNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(NotNode, self.current, operator, self.e7()) if self.accept('dash'): - return UMinusNode(self.current, self.e7()) + operator = self.create_node(SymbolNode, self.previous) + return self.create_node(UMinusNode, self.current, operator, self.e7()) return self.e7() def e7(self) -> BaseNode: left = self.e8() block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rparen', block_start) + rpar = self.create_node(SymbolNode, self.previous) if not isinstance(left, IdNode): raise ParseException('Function call must be applied to plain id', self.getline(), left.lineno, left.colno) assert isinstance(left.value, str) - left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, args) + left = self.create_node(FunctionNode, left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left, lpar, args, rpar) go_again = True while go_again: go_again = False @@ -798,17 +905,23 @@ class Parser: def e8(self) -> BaseNode: block_start = self.current if self.accept('lparen'): + lpar = self.create_node(SymbolNode, block_start) e = self.statement() self.block_expect('rparen', block_start) - return ParenthesizedNode(e, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rpar = self.create_node(SymbolNode, self.previous) + return ParenthesizedNode(lpar, e, rpar, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lbracket'): + lbracket = self.create_node(SymbolNode, block_start) args = self.args() self.block_expect('rbracket', block_start) - return ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(ArrayNode, lbracket, args, rbracket, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) elif self.accept('lcurl'): + lcurl = self.create_node(SymbolNode, block_start) key_values = self.key_values() self.block_expect('rcurl', block_start) - return DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) + rcurl = self.create_node(SymbolNode, self.previous) + return self.create_node(DictNode, lcurl, key_values, rcurl, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) else: return self.e9() @@ -816,34 +929,35 @@ class Parser: t = self.current if self.accept('true'): t.value = True - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('false'): t.value = False - return BooleanNode(t) + return self.create_node(BooleanNode, t) if self.accept('id'): - return IdNode(t) + return self.create_node(IdNode, t) if self.accept('number'): - return NumberNode(t) + return self.create_node(NumberNode, t) if self.accept('string'): - return StringNode(t) + return self.create_node(StringNode, t) if self.accept('fstring'): - return FormatStringNode(t) + return self.create_node(FormatStringNode, t) if self.accept('multiline_string'): - return MultilineStringNode(t) + return self.create_node(MultilineStringNode, t) if self.accept('multiline_fstring'): - return MultilineFormatStringNode(t) + return self.create_node(MultilineFormatStringNode, t) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def key_values(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) a.set_kwarg_no_check(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: raise ParseException('Only key:value pairs are valid in dict construction.', self.getline(), s.lineno, s.colno) @@ -852,20 +966,21 @@ class Parser: def args(self) -> ArgumentNode: s = self.statement() - a = ArgumentNode(self.current) + a = self.create_node(ArgumentNode, self.current) while not isinstance(s, EmptyNode): if self.accept('comma'): - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) a.append(s) elif self.accept('colon'): + a.columns.append(self.create_node(SymbolNode, self.previous)) if not isinstance(s, IdNode): raise ParseException('Dictionary key must be a plain identifier.', self.getline(), s.lineno, s.colno) a.set_kwarg(s, self.statement()) if not self.accept('comma'): return a - a.commas.append(self.previous) + a.commas.append(self.create_node(SymbolNode, self.previous)) else: a.append(s) return a @@ -873,6 +988,7 @@ class Parser: return a def method_call(self, source_object: BaseNode) -> MethodNode: + dot = self.create_node(SymbolNode, self.previous) methodname = self.e9() if not isinstance(methodname, IdNode): if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode): @@ -882,63 +998,78 @@ class Parser: self.getline(), self.current.lineno, self.current.colno) assert isinstance(methodname.value, str) self.expect('lparen') + lpar = self.create_node(SymbolNode, self.previous) args = self.args() + rpar = self.create_node(SymbolNode, self.current) self.expect('rparen') - method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname, args) + method = self.create_node(MethodNode, methodname.filename, methodname.lineno, methodname.colno, + source_object, dot, methodname, lpar, args, rpar) if self.accept('dot'): return self.method_call(method) return method def index_call(self, source_object: BaseNode) -> IndexNode: + lbracket = self.create_node(SymbolNode, self.previous) index_statement = self.statement() self.expect('rbracket') - return IndexNode(source_object, index_statement) + rbracket = self.create_node(SymbolNode, self.previous) + return self.create_node(IndexNode, source_object, lbracket, index_statement, rbracket) def foreachblock(self) -> ForeachClauseNode: + foreach_ = self.create_node(SymbolNode, self.previous) self.expect('id') assert isinstance(self.previous.value, str) - varname = self.previous - varnames = [IdNode(self.previous)] + varnames = [self.create_node(IdNode, self.previous)] + commas = [] if self.accept('comma'): + commas.append(self.create_node(SymbolNode, self.previous)) self.expect('id') assert isinstance(self.previous.value, str) - varnames.append(IdNode(self.previous)) + varnames.append(self.create_node(IdNode, self.previous)) self.expect('colon') + column = self.create_node(SymbolNode, self.previous) items = self.statement() block = self.codeblock() - return ForeachClauseNode(varname, varnames, items, block) + endforeach = self.create_node(SymbolNode, self.current) + return self.create_node(ForeachClauseNode, foreach_, varnames, commas, column, items, block, endforeach) def ifblock(self) -> IfClauseNode: + if_node = self.create_node(SymbolNode, self.previous) condition = self.statement() - clause = IfClauseNode(condition) + clause = self.create_node(IfClauseNode, condition) self.expect('eol') block = self.codeblock() - clause.ifs.append(IfNode(clause, condition, block)) + clause.ifs.append(self.create_node(IfNode, clause, if_node, condition, block)) self.elseifblock(clause) clause.elseblock = self.elseblock() + clause.endif = self.create_node(SymbolNode, self.current) return clause def elseifblock(self, clause: IfClauseNode) -> None: while self.accept('elif'): + elif_ = self.create_node(SymbolNode, self.previous) s = self.statement() self.expect('eol') b = self.codeblock() - clause.ifs.append(IfNode(s, s, b)) + clause.ifs.append(self.create_node(IfNode, s, elif_, s, b)) def elseblock(self) -> T.Union[ElseNode, EmptyNode]: if self.accept('else'): + else_ = self.create_node(SymbolNode, self.previous) self.expect('eol') block = self.codeblock() - return ElseNode(block) + return ElseNode(else_, block) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) def testcaseblock(self) -> TestCaseClauseNode: + testcase = self.create_node(SymbolNode, self.previous) condition = self.statement() self.expect('eol') block = self.codeblock() - return TestCaseClauseNode(condition, block) + endtestcase = SymbolNode(self.current) + return self.create_node(TestCaseClauseNode, testcase, condition, block, endtestcase) def line(self) -> BaseNode: block_start = self.current @@ -953,9 +1084,9 @@ class Parser: self.block_expect('endforeach', block_start) return forblock if self.accept('continue'): - return ContinueNode(self.current) + return self.create_node(ContinueNode, self.current) if self.accept('break'): - return BreakNode(self.current) + return self.create_node(BreakNode, self.current) if self.lexer.in_unit_test and self.accept('testcase'): block = self.testcaseblock() self.block_expect('endtestcase', block_start) @@ -963,15 +1094,20 @@ class Parser: return self.statement() def codeblock(self) -> CodeBlockNode: - block = CodeBlockNode(self.current) + block = self.create_node(CodeBlockNode, self.current) cond = True + try: while cond: curline = self.line() + if not isinstance(curline, EmptyNode): block.lines.append(curline) + cond = self.accept('eol') + except ParseException as e: e.ast = block raise + return block diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py index af3210cf3..64eb2f2f1 100644 --- a/mesonbuild/rewriter.py +++ b/mesonbuild/rewriter.py @@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL from mesonbuild.mesonlib import MesonException, setup_vsenv from . import mlog, environment from functools import wraps -from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode +from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode import json, os, re, sys import typing as T @@ -104,6 +104,9 @@ class RequiredKeys: return wrapped +def _symbol(val: str) -> SymbolNode: + return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val)) + class MTypeBase: def __init__(self, node: T.Optional[BaseNode] = None): if node is None: @@ -189,7 +192,7 @@ class MTypeList(MTypeBase): super().__init__(node) def _new_node(self): - return ArrayNode(ArgumentNode(Token('', '', 0, 0, 0, None, '')), 0, 0, 0, 0) + return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'), 0, 0, 0, 0) def _new_element_node(self, value): # Overwrite in derived class @@ -728,7 +731,7 @@ class Rewriter: node = tgt_function.args.kwargs[extra_files_key] except StopIteration: # Target has no extra_files kwarg, create one - node = ArrayNode(ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) + node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node mark_array = False if tgt_function not in self.modified_nodes: @@ -812,17 +815,17 @@ class Rewriter: # Build src list src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) + src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'), 0, 0, 0, 0) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), src_far_node) - src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), src_fun_node) + src_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')')) + src_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node) src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_far_node.arguments = [src_arr_node] # Build target tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) - tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), tgt_arg_node) - tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), tgt_fun_node) + tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')')) + tgt_ass_node = AssignmentNode(filename, 0, 0, IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node) tgt_arg_node.arguments = [ StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), IdNode(Token('string', filename, 0, 0, 0, None, source_id))