Chiebot-Mirror
/
meson
mirror of https://github.com/mesonbuild/meson.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #12152 from bruchar1/ast-preserve-all

Browse Source 
Preserve whitespaces and comments in AST
pull/12249/head
Jussi Pakkanen 1 year ago committed by GitHub
parent 1b6c9ad02a 14e35b63c0
commit d2dfef5205
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 920 additions and 272 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      docs/markdown/Syntax.md
  2. 27
      mesonbuild/ast/interpreter.py
  3. 10
      mesonbuild/ast/introspection.py
  4. 251
      mesonbuild/ast/printer.py
  5. 26
      mesonbuild/ast/visitor.py
  6. 22
      mesonbuild/cargo/builder.py
  7. 18
      mesonbuild/cmake/interpreter.py
  8. 2
      mesonbuild/coredata.py
  9. 6
      mesonbuild/interpreter/interpreter.py
  10. 4
      mesonbuild/interpreterbase/helpers.py
  11. 40
      mesonbuild/interpreterbase/interpreterbase.py
  12. 6
      mesonbuild/mintro.py
  13. 509
      mesonbuild/mparser.py
  14. 8
      mesonbuild/optinterpreter.py
  15. 47
      mesonbuild/rewriter.py
  16. 1
      run_format_tests.py
  17. 189
      test cases/unit/118 rewrite/meson.build
  18. 22
      unittests/rewritetests.py

2
docs/markdown/Syntax.md
Unescape View File

@ -109,7 +109,7 @@ Strings in Meson are declared with single quotes. To enter a literal
single quote do it like this: single quote do it like this:
```meson ```meson
single quote = 'contains a \' character' single_quote = 'contains a \' character'
``` ```
The full list of escape sequences is: The full list of escape sequences is:

27
mesonbuild/ast/interpreter.py
Unescape View File

@ -239,7 +239,7 @@ class AstInterpreter(InterpreterBase):
def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs: def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs:
def resolve_key(node: mparser.BaseNode) -> str: def resolve_key(node: mparser.BaseNode) -> str:
if isinstance(node, mparser.StringNode): if isinstance(node, mparser.BaseStringNode):
return node.value return node.value
return '__AST_UNKNOWN__' return '__AST_UNKNOWN__'
arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key) arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key)
@ -254,10 +254,10 @@ class AstInterpreter(InterpreterBase):
def evaluate_plusassign(self, node: PlusAssignmentNode) -> None: def evaluate_plusassign(self, node: PlusAssignmentNode) -> None:
assert isinstance(node, PlusAssignmentNode) assert isinstance(node, PlusAssignmentNode)
# Cheat by doing a reassignment # Cheat by doing a reassignment
self.assignments[node.var_name] = node.value # Save a reference to the value node self.assignments[node.var_name.value] = node.value # Save a reference to the value node
if node.value.ast_id: if node.value.ast_id:
self.reverse_assignment[node.value.ast_id] = node self.reverse_assignment[node.value.ast_id] = node
self.assign_vals[node.var_name] = self.evaluate_statement(node.value) self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value)
def evaluate_indexing(self, node: IndexNode) -> int: def evaluate_indexing(self, node: IndexNode) -> int:
return 0 return 0
@ -312,17 +312,17 @@ class AstInterpreter(InterpreterBase):
for i in node.ifs: for i in node.ifs:
self.evaluate_codeblock(i.block) self.evaluate_codeblock(i.block)
if not isinstance(node.elseblock, EmptyNode): if not isinstance(node.elseblock, EmptyNode):
self.evaluate_codeblock(node.elseblock) self.evaluate_codeblock(node.elseblock.block)
def get_variable(self, varname: str) -> int: def get_variable(self, varname: str) -> int:
return 0 return 0
def assignment(self, node: AssignmentNode) -> None: def assignment(self, node: AssignmentNode) -> None:
assert isinstance(node, AssignmentNode) assert isinstance(node, AssignmentNode)
self.assignments[node.var_name] = node.value # Save a reference to the value node self.assignments[node.var_name.value] = node.value # Save a reference to the value node
if node.value.ast_id: if node.value.ast_id:
self.reverse_assignment[node.value.ast_id] = node self.reverse_assignment[node.value.ast_id] = node
self.assign_vals[node.var_name] = self.evaluate_statement(node.value) # Evaluate the value just in case self.assign_vals[node.var_name.value] = self.evaluate_statement(node.value) # Evaluate the value just in case
def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]: def resolve_node(self, node: BaseNode, include_unknown_args: bool = False, id_loop_detect: T.Optional[T.List[str]] = None) -> T.Optional[T.Any]:
def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any: def quick_resolve(n: BaseNode, loop_detect: T.Optional[T.List[str]] = None) -> T.Any:
@ -371,8 +371,8 @@ class AstInterpreter(InterpreterBase):
elif isinstance(node, ArithmeticNode): elif isinstance(node, ArithmeticNode):
if node.operation != 'add': if node.operation != 'add':
return None # Only handle string and array concats return None # Only handle string and array concats
l = quick_resolve(node.left) l = self.resolve_node(node.left, include_unknown_args, id_loop_detect)
r = quick_resolve(node.right) r = self.resolve_node(node.right, include_unknown_args, id_loop_detect)
if isinstance(l, str) and isinstance(r, str): if isinstance(l, str) and isinstance(r, str):
result = l + r # String concatenation detected result = l + r # String concatenation detected
else: else:
@ -382,17 +382,18 @@ class AstInterpreter(InterpreterBase):
src = quick_resolve(node.source_object) src = quick_resolve(node.source_object)
margs = self.flatten_args(node.args.arguments, include_unknown_args, id_loop_detect) margs = self.flatten_args(node.args.arguments, include_unknown_args, id_loop_detect)
mkwargs: T.Dict[str, TYPE_nvar] = {} mkwargs: T.Dict[str, TYPE_nvar] = {}
method_name = node.name.value
try: try:
if isinstance(src, str): if isinstance(src, str):
result = StringHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) result = StringHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs)
elif isinstance(src, bool): elif isinstance(src, bool):
result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) result = BooleanHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs)
elif isinstance(src, int): elif isinstance(src, int):
result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) result = IntegerHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs)
elif isinstance(src, list): elif isinstance(src, list):
result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) result = ArrayHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs)
elif isinstance(src, dict): elif isinstance(src, dict):
result = DictHolder(src, T.cast('Interpreter', self)).method_call(node.name, margs, mkwargs) result = DictHolder(src, T.cast('Interpreter', self)).method_call(method_name, margs, mkwargs)
except mesonlib.MesonException: except mesonlib.MesonException:
return None return None

10
mesonbuild/ast/introspection.py
Unescape View File

@ -27,7 +27,7 @@ from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary
from ..compilers import detect_compiler_for from ..compilers import detect_compiler_for
from ..interpreterbase import InvalidArguments from ..interpreterbase import InvalidArguments
from ..mesonlib import MachineChoice, OptionKey from ..mesonlib import MachineChoice, OptionKey
from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode
from .interpreter import AstInterpreter from .interpreter import AstInterpreter
if T.TYPE_CHECKING: if T.TYPE_CHECKING:
@ -128,7 +128,7 @@ class IntrospectionInterpreter(AstInterpreter):
if not self.is_subproject() and 'subproject_dir' in kwargs: if not self.is_subproject() and 'subproject_dir' in kwargs:
spdirname = kwargs['subproject_dir'] spdirname = kwargs['subproject_dir']
if isinstance(spdirname, StringNode): if isinstance(spdirname, BaseStringNode):
assert isinstance(spdirname.value, str) assert isinstance(spdirname.value, str)
self.subproject_dir = spdirname.value self.subproject_dir = spdirname.value
if not self.is_subproject(): if not self.is_subproject():
@ -174,7 +174,7 @@ class IntrospectionInterpreter(AstInterpreter):
for l in self.flatten_args(raw_langs): for l in self.flatten_args(raw_langs):
if isinstance(l, str): if isinstance(l, str):
langs.append(l) langs.append(l)
elif isinstance(l, StringNode): elif isinstance(l, BaseStringNode):
langs.append(l.value) langs.append(l.value)
for lang in sorted(langs, key=compilers.sort_clink): for lang in sorted(langs, key=compilers.sort_clink):
@ -261,9 +261,9 @@ class IntrospectionInterpreter(AstInterpreter):
continue continue
arg_nodes = arg_node.arguments.copy() arg_nodes = arg_node.arguments.copy()
# Pop the first element if the function is a build target function # Pop the first element if the function is a build target function
if isinstance(curr, FunctionNode) and curr.func_name in BUILD_TARGET_FUNCTIONS: if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS:
arg_nodes.pop(0) arg_nodes.pop(0)
elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))] elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))]
inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))] inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))]
if elementary_nodes: if elementary_nodes:
res += [curr] res += [curr]

251
mesonbuild/ast/printer.py
Unescape View File

@ -18,6 +18,8 @@ from __future__ import annotations
from .. import mparser from .. import mparser
from .visitor import AstVisitor from .visitor import AstVisitor
from itertools import zip_longest
import re import re
import typing as T import typing as T
@ -84,7 +86,17 @@ class AstPrinter(AstVisitor):
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
assert isinstance(node.value, str) assert isinstance(node.value, str)
self.append("f'" + node.value + "'", node) self.append("f'" + self.escape(node.value) + "'", node)
node.lineno = self.curr_line or node.lineno
def visit_MultilineStringNode(self, node: mparser.StringNode) -> None:
assert isinstance(node.value, str)
self.append("'''" + node.value + "'''", node)
node.lineno = self.curr_line or node.lineno
def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None:
assert isinstance(node.value, str)
self.append("f'''" + node.value + "'''", node)
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
@ -152,30 +164,30 @@ class AstPrinter(AstVisitor):
def visit_MethodNode(self, node: mparser.MethodNode) -> None: def visit_MethodNode(self, node: mparser.MethodNode) -> None:
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
node.source_object.accept(self) node.source_object.accept(self)
self.append('.' + node.name + '(', node) self.append('.' + node.name.value + '(', node)
node.args.accept(self) node.args.accept(self)
self.append(')', node) self.append(')', node)
def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
self.append(node.func_name + '(', node) self.append(node.func_name.value + '(', node)
node.args.accept(self) node.args.accept(self)
self.append(')', node) self.append(')', node)
def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
self.append(node.var_name + ' = ', node) self.append(node.var_name.value + ' = ', node)
node.value.accept(self) node.value.accept(self)
def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
self.append(node.var_name + ' += ', node) self.append(node.var_name.value + ' += ', node)
node.value.accept(self) node.value.accept(self)
def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
node.lineno = self.curr_line or node.lineno node.lineno = self.curr_line or node.lineno
self.append_padded('foreach', node) self.append_padded('foreach', node)
self.append_padded(', '.join(node.varnames), node) self.append_padded(', '.join(varname.value for varname in node.varnames), node)
self.append_padded(':', node) self.append_padded(':', node)
node.items.accept(self) node.items.accept(self)
self.newline() self.newline()
@ -238,6 +250,223 @@ class AstPrinter(AstVisitor):
else: else:
self.result = re.sub(r', $', '', self.result) self.result = re.sub(r', $', '', self.result)
class RawPrinter(AstVisitor):
def __init__(self):
self.result = ''
def visit_default_func(self, node: mparser.BaseNode):
self.result += node.value
if node.whitespaces:
node.whitespaces.accept(self)
def visit_unary_operator(self, node: mparser.UnaryOperatorNode):
node.operator.accept(self)
node.value.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_binary_operator(self, node: mparser.BinaryOperatorNode):
node.left.accept(self)
node.operator.accept(self)
node.right.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_BooleanNode(self, node: mparser.BooleanNode) -> None:
self.result += 'true' if node.value else 'false'
if node.whitespaces:
node.whitespaces.accept(self)
def visit_NumberNode(self, node: mparser.NumberNode) -> None:
self.result += node.raw_value
if node.whitespaces:
node.whitespaces.accept(self)
def visit_StringNode(self, node: mparser.StringNode) -> None:
self.result += f"'{node.raw_value}'"
if node.whitespaces:
node.whitespaces.accept(self)
def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None:
self.result += f"'''{node.value}'''"
if node.whitespaces:
node.whitespaces.accept(self)
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
self.result += 'f'
self.visit_StringNode(node)
def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None:
self.result += 'f'
self.visit_MultilineStringNode(node)
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
self.result += 'continue'
if node.whitespaces:
node.whitespaces.accept(self)
def visit_BreakNode(self, node: mparser.BreakNode) -> None:
self.result += 'break'
if node.whitespaces:
node.whitespaces.accept(self)
def visit_ArrayNode(self, node: mparser.ArrayNode) -> None:
node.lbracket.accept(self)
node.args.accept(self)
node.rbracket.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_DictNode(self, node: mparser.DictNode) -> None:
node.lcurl.accept(self)
node.args.accept(self)
node.rcurl.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None:
node.lpar.accept(self)
node.inner.accept(self)
node.rpar.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_OrNode(self, node: mparser.OrNode) -> None:
self.visit_binary_operator(node)
def visit_AndNode(self, node: mparser.AndNode) -> None:
self.visit_binary_operator(node)
def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None:
self.visit_binary_operator(node)
def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None:
self.visit_binary_operator(node)
def visit_NotNode(self, node: mparser.NotNode) -> None:
self.visit_unary_operator(node)
def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None:
if node.pre_whitespaces:
node.pre_whitespaces.accept(self)
for i in node.lines:
i.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_IndexNode(self, node: mparser.IndexNode) -> None:
node.iobject.accept(self)
node.lbracket.accept(self)
node.index.accept(self)
node.rbracket.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_MethodNode(self, node: mparser.MethodNode) -> None:
node.source_object.accept(self)
node.dot.accept(self)
node.name.accept(self)
node.lpar.accept(self)
node.args.accept(self)
node.rpar.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
node.func_name.accept(self)
node.lpar.accept(self)
node.args.accept(self)
node.rpar.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
node.var_name.accept(self)
node.operator.accept(self)
node.value.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
node.var_name.accept(self)
node.operator.accept(self)
node.value.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
node.foreach_.accept(self)
for varname, comma in zip_longest(node.varnames, node.commas):
varname.accept(self)
if comma is not None:
comma.accept(self)
node.column.accept(self)
node.items.accept(self)
node.block.accept(self)
node.endforeach.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None:
for i in node.ifs:
i.accept(self)
if not isinstance(node.elseblock, mparser.EmptyNode):
node.elseblock.accept(self)
node.endif.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_UMinusNode(self, node: mparser.UMinusNode) -> None:
self.visit_unary_operator(node)
def visit_IfNode(self, node: mparser.IfNode) -> None:
node.if_.accept(self)
node.condition.accept(self)
node.block.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_ElseNode(self, node: mparser.ElseNode) -> None:
node.else_.accept(self)
node.block.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_TernaryNode(self, node: mparser.TernaryNode) -> None:
node.condition.accept(self)
node.questionmark.accept(self)
node.trueblock.accept(self)
node.column.accept(self)
node.falseblock.accept(self)
if node.whitespaces:
node.whitespaces.accept(self)
def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None:
commas_iter = iter(node.commas)
for arg in node.arguments:
arg.accept(self)
try:
comma = next(commas_iter)
comma.accept(self)
except StopIteration:
pass
assert len(node.columns) == len(node.kwargs)
for (key, val), column in zip(node.kwargs.items(), node.columns):
key.accept(self)
column.accept(self)
val.accept(self)
try:
comma = next(commas_iter)
comma.accept(self)
except StopIteration:
pass
if node.whitespaces:
node.whitespaces.accept(self)
class AstJSONPrinter(AstVisitor): class AstJSONPrinter(AstVisitor):
def __init__(self) -> None: def __init__(self) -> None:
self.result: T.Dict[str, T.Any] = {} self.result: T.Dict[str, T.Any] = {}
@ -342,28 +571,28 @@ class AstJSONPrinter(AstVisitor):
def visit_MethodNode(self, node: mparser.MethodNode) -> None: def visit_MethodNode(self, node: mparser.MethodNode) -> None:
self._accept('object', node.source_object) self._accept('object', node.source_object)
self._accept('args', node.args) self._accept('args', node.args)
self.current['name'] = node.name self.current['name'] = node.name.value
self.setbase(node) self.setbase(node)
def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
self._accept('args', node.args) self._accept('args', node.args)
self.current['name'] = node.func_name self.current['name'] = node.func_name.value
self.setbase(node) self.setbase(node)
def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
self._accept('value', node.value) self._accept('value', node.value)
self.current['var_name'] = node.var_name self.current['var_name'] = node.var_name.value
self.setbase(node) self.setbase(node)
def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
self._accept('value', node.value) self._accept('value', node.value)
self.current['var_name'] = node.var_name self.current['var_name'] = node.var_name.value
self.setbase(node) self.setbase(node)
def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
self._accept('items', node.items) self._accept('items', node.items)
self._accept('block', node.block) self._accept('block', node.block)
self.current['varnames'] = node.varnames self.current['varnames'] = [varname.value for varname in node.varnames]
self.setbase(node) self.setbase(node)
def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None: def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None:

26
mesonbuild/ast/visitor.py
Unescape View File

@ -43,12 +43,24 @@ class AstVisitor:
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None: def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
def visit_MultilineStringNode(self, node: mparser.StringNode) -> None:
self.visit_default_func(node)
def visit_FormatMultilineStringNode(self, node: mparser.FormatStringNode) -> None:
self.visit_default_func(node)
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None: def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
def visit_BreakNode(self, node: mparser.BreakNode) -> None: def visit_BreakNode(self, node: mparser.BreakNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
def visit_SymbolNode(self, node: mparser.SymbolNode) -> None:
self.visit_default_func(node)
def visit_WhitespaceNode(self, node: mparser.WhitespaceNode) -> None:
self.visit_default_func(node)
def visit_ArrayNode(self, node: mparser.ArrayNode) -> None: def visit_ArrayNode(self, node: mparser.ArrayNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.args.accept(self) node.args.accept(self)
@ -97,22 +109,28 @@ class AstVisitor:
def visit_MethodNode(self, node: mparser.MethodNode) -> None: def visit_MethodNode(self, node: mparser.MethodNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.source_object.accept(self) node.source_object.accept(self)
node.name.accept(self)
node.args.accept(self) node.args.accept(self)
def visit_FunctionNode(self, node: mparser.FunctionNode) -> None: def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.func_name.accept(self)
node.args.accept(self) node.args.accept(self)
def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None: def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.var_name.accept(self)
node.value.accept(self) node.value.accept(self)
def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None: def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.var_name.accept(self)
node.value.accept(self) node.value.accept(self)
def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None: def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
for varname in node.varnames:
varname.accept(self)
node.items.accept(self) node.items.accept(self)
node.block.accept(self) node.block.accept(self)
@ -131,6 +149,10 @@ class AstVisitor:
node.condition.accept(self) node.condition.accept(self)
node.block.accept(self) node.block.accept(self)
def visit_ElseNode(self, node: mparser.IfNode) -> None:
self.visit_default_func(node)
node.block.accept(self)
def visit_TernaryNode(self, node: mparser.TernaryNode) -> None: def visit_TernaryNode(self, node: mparser.TernaryNode) -> None:
self.visit_default_func(node) self.visit_default_func(node)
node.condition.accept(self) node.condition.accept(self)
@ -144,3 +166,7 @@ class AstVisitor:
for key, val in node.kwargs.items(): for key, val in node.kwargs.items():
key.accept(self) key.accept(self)
val.accept(self) val.accept(self)
def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None:
self.visit_default_func(node)
node.inner.accept(self)

22
mesonbuild/cargo/builder.py
Unescape View File

@ -28,6 +28,10 @@ def _token(tid: str, filename: str, value: mparser.TV_TokenTypes) -> mparser.Tok
return mparser.Token(tid, filename, -1, -1, -1, (-1, -1), value) return mparser.Token(tid, filename, -1, -1, -1, (-1, -1), value)
def _symbol(filename: str, val: str) -> mparser.SymbolNode:
return mparser.SymbolNode(_token('', filename, val))
def string(value: str, filename: str) -> mparser.StringNode: def string(value: str, filename: str) -> mparser.StringNode:
"""Build A StringNode """Build A StringNode
@ -45,7 +49,7 @@ def number(value: int, filename: str) -> mparser.NumberNode:
:param filename: the file that the value came from :param filename: the file that the value came from
:return: A NumberNode :return: A NumberNode
""" """
return mparser.NumberNode(_token('number', filename, value)) return mparser.NumberNode(_token('number', filename, str(value)))
def bool(value: builtins.bool, filename: str) -> mparser.BooleanNode: def bool(value: builtins.bool, filename: str) -> mparser.BooleanNode:
@ -67,7 +71,7 @@ def array(value: T.List[mparser.BaseNode], filename: str) -> mparser.ArrayNode:
""" """
args = mparser.ArgumentNode(_token('array', filename, 'unused')) args = mparser.ArgumentNode(_token('array', filename, 'unused'))
args.arguments = value args.arguments = value
return mparser.ArrayNode(args, -1, -1, -1, -1) return mparser.ArrayNode(_symbol(filename, '['), args, _symbol(filename, ']'))
def identifier(value: str, filename: str) -> mparser.IdNode: def identifier(value: str, filename: str) -> mparser.IdNode:
@ -97,7 +101,7 @@ def method(name: str, id_: mparser.IdNode,
args.arguments = pos args.arguments = pos
if kw is not None: if kw is not None:
args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()} args.kwargs = {identifier(k, id_.filename): v for k, v in kw.items()}
return mparser.MethodNode(id_.filename, -1, -1, id_, name, args) return mparser.MethodNode(id_, _symbol(id_.filename, '.'), identifier(name, id_.filename), _symbol(id_.filename, '('), args, _symbol(id_.filename, ')'))
def function(name: str, filename: str, def function(name: str, filename: str,
@ -117,7 +121,7 @@ def function(name: str, filename: str,
args.arguments = pos args.arguments = pos
if kw is not None: if kw is not None:
args.kwargs = {identifier(k, filename): v for k, v in kw.items()} args.kwargs = {identifier(k, filename): v for k, v in kw.items()}
return mparser.FunctionNode(filename, -1, -1, -1, -1, name, args) return mparser.FunctionNode(identifier(name, filename), _symbol(filename, '('), args, _symbol(filename, ')'))
def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode: def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNode:
@ -127,7 +131,7 @@ def equal(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.ComparisonNod
:param rhs: the right hand side of the equal :param rhs: the right hand side of the equal
:return: A compraison node :return: A compraison node
""" """
return mparser.ComparisonNode('==', lhs, rhs) return mparser.ComparisonNode('==', lhs, _symbol(lhs.filename, '=='), rhs)
def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode: def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode:
@ -137,7 +141,7 @@ def or_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.OrNode:
:param rhs: The Right of the Node :param rhs: The Right of the Node
:return: The OrNode :return: The OrNode
""" """
return mparser.OrNode(lhs, rhs) return mparser.OrNode(lhs, _symbol(lhs.filename, 'or'), rhs)
def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode: def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode:
@ -147,7 +151,7 @@ def and_(lhs: mparser.BaseNode, rhs: mparser.BaseNode) -> mparser.AndNode:
:param rhs: The right of the And :param rhs: The right of the And
:return: The AndNode :return: The AndNode
""" """
return mparser.AndNode(lhs, rhs) return mparser.AndNode(lhs, _symbol(lhs.filename, 'and'), rhs)
def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode: def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode:
@ -157,7 +161,7 @@ def not_(value: mparser.BaseNode, filename: str) -> mparser.NotNode:
:param filename: the string filename :param filename: the string filename
:return: The NotNode :return: The NotNode
""" """
return mparser.NotNode(_token('not', filename, ''), value) return mparser.NotNode(_token('not', filename, ''), _symbol(filename, 'not'), value)
def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.AssignmentNode: def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.AssignmentNode:
@ -168,7 +172,7 @@ def assign(value: mparser.BaseNode, varname: str, filename: str) -> mparser.Assi
:param filename: The filename :param filename: The filename
:return: An AssignmentNode :return: An AssignmentNode
""" """
return mparser.AssignmentNode(filename, -1, -1, varname, value) return mparser.AssignmentNode(identifier(varname, filename), _symbol(filename, '='), value)
def block(filename: str) -> mparser.CodeBlockNode: def block(filename: str) -> mparser.CodeBlockNode:

18
mesonbuild/cmake/interpreter.py
Unescape View File

@ -48,6 +48,7 @@ from ..mparser import (
IndexNode, IndexNode,
MethodNode, MethodNode,
NumberNode, NumberNode,
SymbolNode,
) )
@ -959,14 +960,17 @@ class CMakeInterpreter:
def token(tid: str = 'string', val: TYPE_mixed = '') -> Token: def token(tid: str = 'string', val: TYPE_mixed = '') -> Token:
return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val)
def symbol(val: str) -> SymbolNode:
return SymbolNode(token('', val))
def string(value: str) -> StringNode: def string(value: str) -> StringNode:
return StringNode(token(val=value)) return StringNode(token(val=value), escape=False)
def id_node(value: str) -> IdNode: def id_node(value: str) -> IdNode:
return IdNode(token(val=value)) return IdNode(token(val=value))
def number(value: int) -> NumberNode: def number(value: int) -> NumberNode:
return NumberNode(token(val=value)) return NumberNode(token(val=str(value)))
def nodeify(value: TYPE_mixed_list) -> BaseNode: def nodeify(value: TYPE_mixed_list) -> BaseNode:
if isinstance(value, str): if isinstance(value, str):
@ -984,14 +988,14 @@ class CMakeInterpreter:
raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value))) raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value)))
def indexed(node: BaseNode, index: int) -> IndexNode: def indexed(node: BaseNode, index: int) -> IndexNode:
return IndexNode(node, nodeify(index)) return IndexNode(node, symbol('['), nodeify(index), symbol(']'))
def array(elements: TYPE_mixed_list) -> ArrayNode: def array(elements: TYPE_mixed_list) -> ArrayNode:
args = ArgumentNode(token()) args = ArgumentNode(token())
if not isinstance(elements, list): if not isinstance(elements, list):
elements = [args] elements = [args]
args.arguments += [nodeify(x) for x in elements if x is not None] args.arguments += [nodeify(x) for x in elements if x is not None]
return ArrayNode(args, 0, 0, 0, 0) return ArrayNode(symbol('['), args, symbol(']'))
def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode:
args = [] if args is None else args args = [] if args is None else args
@ -1002,7 +1006,7 @@ class CMakeInterpreter:
args = [args] args = [args]
args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.arguments = [nodeify(x) for x in args if x is not None]
args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None}
func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, name, args_n) func_n = FunctionNode(id_node(name), symbol('('), args_n, symbol(')'))
return func_n return func_n
def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode:
@ -1014,10 +1018,10 @@ class CMakeInterpreter:
args = [args] args = [args]
args_n.arguments = [nodeify(x) for x in args if x is not None] args_n.arguments = [nodeify(x) for x in args if x is not None]
args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None}
return MethodNode(self.subdir.as_posix(), 0, 0, obj, name, args_n) return MethodNode(obj, symbol('.'), id_node(name), symbol('('), args_n, symbol(')'))
def assign(var_name: str, value: BaseNode) -> AssignmentNode: def assign(var_name: str, value: BaseNode) -> AssignmentNode:
return AssignmentNode(self.subdir.as_posix(), 0, 0, var_name, value) return AssignmentNode(id_node(var_name), symbol('='), value)
# Generate the root code block and the project function call # Generate the root code block and the project function call
root_cb = CodeBlockNode(token()) root_cb = CodeBlockNode(token())

2
mesonbuild/coredata.py
Unescape View File

@ -1098,7 +1098,7 @@ class MachineFileParser():
return section return section
def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]: def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
if isinstance(node, (mparser.StringNode)): if isinstance(node, (mparser.BaseStringNode)):
return node.value return node.value
elif isinstance(node, mparser.BooleanNode): elif isinstance(node, mparser.BooleanNode):
return node.value return node.value

6
mesonbuild/interpreter/interpreter.py
Unescape View File

@ -536,7 +536,7 @@ class Interpreter(InterpreterBase, HoldableObject):
assert isinstance(kw, mparser.IdNode), 'for mypy' assert isinstance(kw, mparser.IdNode), 'for mypy'
if kw.value == 'meson_version': if kw.value == 'meson_version':
# mypy does not understand "and isinstance" # mypy does not understand "and isinstance"
if isinstance(val, mparser.StringNode): if isinstance(val, mparser.BaseStringNode):
self.handle_meson_version(val.value, val) self.handle_meson_version(val.value, val)
def get_build_def_files(self) -> mesonlib.OrderedSet[str]: def get_build_def_files(self) -> mesonlib.OrderedSet[str]:
@ -2942,7 +2942,7 @@ class Interpreter(InterpreterBase, HoldableObject):
def _add_global_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], def _add_global_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]],
args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None:
if self.is_subproject(): if self.is_subproject():
msg = f'Function \'{node.func_name}\' cannot be used in subprojects because ' \ msg = f'Function \'{node.func_name.value}\' cannot be used in subprojects because ' \
'there is no way to make that reliable.\nPlease only call ' \ 'there is no way to make that reliable.\nPlease only call ' \
'this if is_subproject() returns false. Alternatively, ' \ 'this if is_subproject() returns false. Alternatively, ' \
'define a variable that\ncontains your language-specific ' \ 'define a variable that\ncontains your language-specific ' \
@ -2962,7 +2962,7 @@ class Interpreter(InterpreterBase, HoldableObject):
def _add_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]], def _add_arguments(self, node: mparser.FunctionNode, argsdict: T.Dict[str, T.List[str]],
args_frozen: bool, args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None: args_frozen: bool, args: T.List[str], kwargs: 'kwtypes.FuncAddProjectArgs') -> None:
if args_frozen: if args_frozen:
msg = f'Tried to use \'{node.func_name}\' after a build target has been declared.\n' \ msg = f'Tried to use \'{node.func_name.value}\' after a build target has been declared.\n' \
'This is not permitted. Please declare all arguments before your targets.' 'This is not permitted. Please declare all arguments before your targets.'
raise InvalidCode(msg) raise InvalidCode(msg)

4
mesonbuild/interpreterbase/helpers.py
Unescape View File

@ -25,7 +25,7 @@ if T.TYPE_CHECKING:
from .baseobjects import TYPE_var, TYPE_kwargs, SubProject from .baseobjects import TYPE_var, TYPE_kwargs, SubProject
def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']: def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']:
if isinstance(args, mparser.StringNode): if isinstance(args, mparser.BaseStringNode):
assert isinstance(args.value, str) assert isinstance(args.value, str)
return [args.value] return [args.value]
if not isinstance(args, collections.abc.Sequence): if not isinstance(args, collections.abc.Sequence):
@ -35,7 +35,7 @@ def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']
if isinstance(a, list): if isinstance(a, list):
rest = flatten(a) rest = flatten(a)
result = result + rest result = result + rest
elif isinstance(a, mparser.StringNode): elif isinstance(a, mparser.BaseStringNode):
result.append(a.value) result.append(a.value)
else: else:
result.append(a) result.append(a)

40
mesonbuild/interpreterbase/interpreterbase.py
Unescape View File

@ -137,7 +137,7 @@ class InterpreterBase:
if not self.ast.lines: if not self.ast.lines:
raise InvalidCode('No statements in code.') raise InvalidCode('No statements in code.')
first = self.ast.lines[0] first = self.ast.lines[0]
if not isinstance(first, mparser.FunctionNode) or first.func_name != 'project': if not isinstance(first, mparser.FunctionNode) or first.func_name.value != 'project':
p = pathlib.Path(self.source_root).resolve() p = pathlib.Path(self.source_root).resolve()
found = p found = p
for parent in p.parents: for parent in p.parents:
@ -192,11 +192,18 @@ class InterpreterBase:
self.current_node = cur self.current_node = cur
if isinstance(cur, mparser.FunctionNode): if isinstance(cur, mparser.FunctionNode):
return self.function_call(cur) return self.function_call(cur)
elif isinstance(cur, mparser.PlusAssignmentNode):
self.evaluate_plusassign(cur)
elif isinstance(cur, mparser.AssignmentNode): elif isinstance(cur, mparser.AssignmentNode):
self.assignment(cur) self.assignment(cur)
elif isinstance(cur, mparser.MethodNode): elif isinstance(cur, mparser.MethodNode):
return self.method_call(cur) return self.method_call(cur)
elif isinstance(cur, mparser.StringNode): elif isinstance(cur, mparser.BaseStringNode):
if isinstance(cur, mparser.MultilineFormatStringNode):
return self.evaluate_multiline_fstring(cur)
elif isinstance(cur, mparser.FormatStringNode):
return self.evaluate_fstring(cur)
else:
return self._holderify(cur.value) return self._holderify(cur.value)
elif isinstance(cur, mparser.BooleanNode): elif isinstance(cur, mparser.BooleanNode):
return self._holderify(cur.value) return self._holderify(cur.value)
@ -224,21 +231,16 @@ class InterpreterBase:
return self.evaluate_arithmeticstatement(cur) return self.evaluate_arithmeticstatement(cur)
elif isinstance(cur, mparser.ForeachClauseNode): elif isinstance(cur, mparser.ForeachClauseNode):
self.evaluate_foreach(cur) self.evaluate_foreach(cur)
elif isinstance(cur, mparser.PlusAssignmentNode):
self.evaluate_plusassign(cur)
elif isinstance(cur, mparser.IndexNode): elif isinstance(cur, mparser.IndexNode):
return self.evaluate_indexing(cur) return self.evaluate_indexing(cur)
elif isinstance(cur, mparser.TernaryNode): elif isinstance(cur, mparser.TernaryNode):
return self.evaluate_ternary(cur) return self.evaluate_ternary(cur)
elif isinstance(cur, mparser.FormatStringNode):
if isinstance(cur, mparser.MultilineFormatStringNode):
return self.evaluate_multiline_fstring(cur)
else:
return self.evaluate_fstring(cur)
elif isinstance(cur, mparser.ContinueNode): elif isinstance(cur, mparser.ContinueNode):
raise ContinueRequest() raise ContinueRequest()
elif isinstance(cur, mparser.BreakNode): elif isinstance(cur, mparser.BreakNode):
raise BreakRequest() raise BreakRequest()
elif isinstance(cur, mparser.ParenthesizedNode):
return self.evaluate_statement(cur.inner)
elif isinstance(cur, mparser.TestCaseClauseNode): elif isinstance(cur, mparser.TestCaseClauseNode):
return self.evaluate_testcase(cur) return self.evaluate_testcase(cur)
else: else:
@ -254,7 +256,7 @@ class InterpreterBase:
@FeatureNew('dict', '0.47.0') @FeatureNew('dict', '0.47.0')
def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject: def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject:
def resolve_key(key: mparser.BaseNode) -> str: def resolve_key(key: mparser.BaseNode) -> str:
if not isinstance(key, mparser.StringNode): if not isinstance(key, mparser.BaseStringNode):
FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject) FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject)
key_holder = self.evaluate_statement(key) key_holder = self.evaluate_statement(key)
if key_holder is None: if key_holder is None:
@ -301,7 +303,7 @@ class InterpreterBase:
mesonlib.project_meson_versions[self.subproject] = prev_meson_version mesonlib.project_meson_versions[self.subproject] = prev_meson_version
return None return None
if not isinstance(node.elseblock, mparser.EmptyNode): if not isinstance(node.elseblock, mparser.EmptyNode):
self.evaluate_codeblock(node.elseblock) self.evaluate_codeblock(node.elseblock.block)
return None return None
def evaluate_testcase(self, node: mparser.TestCaseClauseNode) -> T.Optional[Disabler]: def evaluate_testcase(self, node: mparser.TestCaseClauseNode) -> T.Optional[Disabler]:
@ -426,9 +428,7 @@ class InterpreterBase:
return self.evaluate_fstring(node) return self.evaluate_fstring(node)
@FeatureNew('format strings', '0.58.0') @FeatureNew('format strings', '0.58.0')
def evaluate_fstring(self, node: mparser.FormatStringNode) -> InterpreterObject: def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> InterpreterObject:
assert isinstance(node, mparser.FormatStringNode)
def replace(match: T.Match[str]) -> str: def replace(match: T.Match[str]) -> str:
var = str(match.group(1)) var = str(match.group(1))
try: try:
@ -459,14 +459,14 @@ class InterpreterBase:
if tsize is None: if tsize is None:
if isinstance(i, tuple): if isinstance(i, tuple):
raise mesonlib.MesonBugException(f'Iteration of {items} returned a tuple even though iter_tuple_size() is None') raise mesonlib.MesonBugException(f'Iteration of {items} returned a tuple even though iter_tuple_size() is None')
self.set_variable(node.varnames[0], self._holderify(i)) self.set_variable(node.varnames[0].value, self._holderify(i))
else: else:
if not isinstance(i, tuple): if not isinstance(i, tuple):
raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}')
if len(i) != tsize: if len(i) != tsize:
raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}') raise mesonlib.MesonBugException(f'Iteration of {items} did not return a tuple even though iter_tuple_size() is {tsize}')
for j in range(tsize): for j in range(tsize):
self.set_variable(node.varnames[j], self._holderify(i[j])) self.set_variable(node.varnames[j].value, self._holderify(i[j]))
try: try:
self.evaluate_codeblock(node.block) self.evaluate_codeblock(node.block)
except ContinueRequest: except ContinueRequest:
@ -476,7 +476,7 @@ class InterpreterBase:
def evaluate_plusassign(self, node: mparser.PlusAssignmentNode) -> None: def evaluate_plusassign(self, node: mparser.PlusAssignmentNode) -> None:
assert isinstance(node, mparser.PlusAssignmentNode) assert isinstance(node, mparser.PlusAssignmentNode)
varname = node.var_name varname = node.var_name.value
addition = self.evaluate_statement(node.value) addition = self.evaluate_statement(node.value)
if addition is None: if addition is None:
raise InvalidCodeOnVoid('plus assign') raise InvalidCodeOnVoid('plus assign')
@ -504,7 +504,7 @@ class InterpreterBase:
return self._holderify(iobject.operator_call(MesonOperator.INDEX, index)) return self._holderify(iobject.operator_call(MesonOperator.INDEX, index))
def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]: def function_call(self, node: mparser.FunctionNode) -> T.Optional[InterpreterObject]:
func_name = node.func_name func_name = node.func_name.value
(h_posargs, h_kwargs) = self.reduce_arguments(node.args) (h_posargs, h_kwargs) = self.reduce_arguments(node.args)
(posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs) (posargs, kwargs) = self._unholder_args(h_posargs, h_kwargs)
if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}: if is_disabled(posargs, kwargs) and func_name not in {'get_variable', 'set_variable', 'unset_variable', 'is_disabler'}:
@ -532,7 +532,7 @@ class InterpreterBase:
else: else:
object_display_name = invocable.__class__.__name__ object_display_name = invocable.__class__.__name__
obj = self.evaluate_statement(invocable) obj = self.evaluate_statement(invocable)
method_name = node.name method_name = node.name.value
(h_args, h_kwargs) = self.reduce_arguments(node.args) (h_args, h_kwargs) = self.reduce_arguments(node.args)
(args, kwargs) = self._unholder_args(h_args, h_kwargs) (args, kwargs) = self._unholder_args(h_args, h_kwargs)
if is_disabled(args, kwargs): if is_disabled(args, kwargs):
@ -628,7 +628,7 @@ class InterpreterBase:
Tried to assign values inside an argument list. Tried to assign values inside an argument list.
To specify a keyword argument, use : instead of =. To specify a keyword argument, use : instead of =.
''')) '''))
var_name = node.var_name var_name = node.var_name.value
if not isinstance(var_name, str): if not isinstance(var_name, str):
raise InvalidArguments('Tried to assign value to a non-variable.') raise InvalidArguments('Tried to assign value to a non-variable.')
value = self.evaluate_statement(node.value) value = self.evaluate_statement(node.value)

6
mesonbuild/mintro.py
Unescape View File

@ -36,7 +36,7 @@ from .dependencies import Dependency
from . import environment from . import environment
from .interpreterbase import ObjectHolder from .interpreterbase import ObjectHolder
from .mesonlib import OptionKey from .mesonlib import OptionKey
from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode
if T.TYPE_CHECKING: if T.TYPE_CHECKING:
import argparse import argparse
@ -187,14 +187,14 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st
args: T.List[BaseNode] = [] args: T.List[BaseNode] = []
if isinstance(n, FunctionNode): if isinstance(n, FunctionNode):
args = list(n.args.arguments) args = list(n.args.arguments)
if n.func_name in BUILD_TARGET_FUNCTIONS: if n.func_name.value in BUILD_TARGET_FUNCTIONS:
args.pop(0) args.pop(0)
elif isinstance(n, ArrayNode): elif isinstance(n, ArrayNode):
args = n.args.arguments args = n.args.arguments
elif isinstance(n, ArgumentNode): elif isinstance(n, ArgumentNode):
args = n.arguments args = n.arguments
for j in args: for j in args:
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
assert isinstance(j.value, str) assert isinstance(j.value, str)
res += [Path(j.value)] res += [Path(j.value)]
elif isinstance(j, str): elif isinstance(j, str):

509
mesonbuild/mparser.py
Unescape View File

@ -26,6 +26,8 @@ if T.TYPE_CHECKING:
from .ast import AstVisitor from .ast import AstVisitor
BaseNodeT = T.TypeVar('BaseNodeT', bound='BaseNode')
# This is the regex for the supported escape sequences of a regular string # This is the regex for the supported escape sequences of a regular string
# literal, like 'abc\x00' # literal, like 'abc\x00'
ESCAPE_SEQUENCE_SINGLE_RE = re.compile(r''' ESCAPE_SEQUENCE_SINGLE_RE = re.compile(r'''
@ -114,12 +116,12 @@ class Lexer:
self.keywords.update({'testcase', 'endtestcase'}) self.keywords.update({'testcase', 'endtestcase'})
self.token_specification = [ self.token_specification = [
# Need to be sorted longest to shortest. # Need to be sorted longest to shortest.
('ignore', re.compile(r'[ \t]')), ('whitespace', re.compile(r'[ \t]+')),
('multiline_fstring', re.compile(r"f'''(.|\n)*?'''", re.M)), ('multiline_fstring', re.compile(r"f'''(.|\n)*?'''", re.M)),
('fstring', re.compile(r"f'([^'\\]|(\\.))*'")), ('fstring', re.compile(r"f'([^'\\]|(\\.))*'")),
('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')), ('id', re.compile('[_a-zA-Z][_0-9a-zA-Z]*')),
('number', re.compile(r'0[bB][01]+|0[oO][0-7]+|0[xX][0-9a-fA-F]+|0|[1-9]\d*')), ('number', re.compile(r'0[bB][01]+|0[oO][0-7]+|0[xX][0-9a-fA-F]+|0|[1-9]\d*')),
('eol_cont', re.compile(r'\\\n')), ('eol_cont', re.compile(r'\\[ \t]*(#.*)?\n')),
('eol', re.compile(r'\n')), ('eol', re.compile(r'\n')),
('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)), ('multiline_string', re.compile(r"'''(.|\n)*?'''", re.M)),
('comment', re.compile(r'#.*')), ('comment', re.compile(r'#.*')),
@ -163,7 +165,7 @@ class Lexer:
col = 0 col = 0
while loc < len(self.code): while loc < len(self.code):
matched = False matched = False
value: T.Union[str, bool, int] = None value: str = ''
for (tid, reg) in self.token_specification: for (tid, reg) in self.token_specification:
mo = reg.match(self.code, loc) mo = reg.match(self.code, loc)
if mo: if mo:
@ -175,10 +177,8 @@ class Lexer:
loc = mo.end() loc = mo.end()
span_end = loc span_end = loc
bytespan = (span_start, span_end) bytespan = (span_start, span_end)
match_text = mo.group() value = mo.group()
if tid in {'ignore', 'comment'}: if tid == 'lparen':
break
elif tid == 'lparen':
par_count += 1 par_count += 1
elif tid == 'rparen': elif tid == 'rparen':
par_count -= 1 par_count -= 1
@ -193,48 +193,34 @@ class Lexer:
elif tid == 'dblquote': elif tid == 'dblquote':
raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col) raise ParseException('Double quotes are not supported. Use single quotes.', self.getline(line_start), lineno, col)
elif tid in {'string', 'fstring'}: elif tid in {'string', 'fstring'}:
# Handle here and not on the regexp to give a better error message. if value.find("\n") != -1:
if match_text.find("\n") != -1:
msg = ("Newline character in a string detected, use ''' (three single quotes) " msg = ("Newline character in a string detected, use ''' (three single quotes) "
"for multiline strings instead.\n" "for multiline strings instead.\n"
"This will become a hard error in a future Meson release.") "This will become a hard error in a future Meson release.")
mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename)) mlog.warning(mlog.code_line(msg, self.getline(line_start), col), location=BaseNode(lineno, col, filename))
value = match_text[2 if tid == 'fstring' else 1:-1] value = value[2 if tid == 'fstring' else 1:-1]
value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, value)
elif tid in {'multiline_string', 'multiline_fstring'}: elif tid in {'multiline_string', 'multiline_fstring'}:
# For multiline strings, parse out the value and pass value = value[4 if tid == 'multiline_fstring' else 3:-3]
# through the normal string logic. lines = value.split('\n')
# For multiline format strings, we have to emit a
# different AST node so we can add a feature check,
# but otherwise, it follows the normal fstring logic.
if tid == 'multiline_string':
value = match_text[3:-3]
tid = 'string'
else:
value = match_text[4:-3]
lines = match_text.split('\n')
if len(lines) > 1: if len(lines) > 1:
lineno += len(lines) - 1 lineno += len(lines) - 1
line_start = mo.end() - len(lines[-1]) line_start = mo.end() - len(lines[-1])
elif tid == 'number':
value = int(match_text, base=0)
elif tid == 'eol_cont': elif tid == 'eol_cont':
lineno += 1 lineno += 1
line_start = loc line_start = loc
break tid = 'whitespace'
elif tid == 'eol': elif tid == 'eol':
lineno += 1 lineno += 1
line_start = loc line_start = loc
if par_count > 0 or bracket_count > 0 or curl_count > 0: if par_count > 0 or bracket_count > 0 or curl_count > 0:
break tid = 'whitespace'
elif tid == 'id': elif tid == 'id':
if match_text in self.keywords: if value in self.keywords:
tid = match_text tid = value
else: else:
if match_text in self.future_keywords: if value in self.future_keywords:
mlog.warning(f"Identifier '{match_text}' will become a reserved keyword in a future release. Please rename it.", mlog.warning(f"Identifier '{value}' will become a reserved keyword in a future release. Please rename it.",
location=BaseNode(lineno, col, filename)) location=BaseNode(lineno, col, filename))
value = match_text
yield Token(tid, filename, curline_start, curline, col, bytespan, value) yield Token(tid, filename, curline_start, curline, col, bytespan, value)
break break
if not matched: if not matched:
@ -247,13 +233,16 @@ class BaseNode:
filename: str = field(hash=False) filename: str = field(hash=False)
end_lineno: int = field(hash=False) end_lineno: int = field(hash=False)
end_colno: int = field(hash=False) end_colno: int = field(hash=False)
whitespaces: T.Optional[WhitespaceNode] = field(hash=False)
def __init__(self, lineno: int, colno: int, filename: str, end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None: def __init__(self, lineno: int, colno: int, filename: str,
end_lineno: T.Optional[int] = None, end_colno: T.Optional[int] = None) -> None:
self.lineno = lineno self.lineno = lineno
self.colno = colno self.colno = colno
self.filename = filename self.filename = filename
self.end_lineno = end_lineno if end_lineno is not None else lineno self.end_lineno = end_lineno if end_lineno is not None else lineno
self.end_colno = end_colno if end_colno is not None else colno self.end_colno = end_colno if end_colno is not None else colno
self.whitespaces = None
# Attributes for the visitors # Attributes for the visitors
self.level = 0 self.level = 0
@ -267,6 +256,26 @@ class BaseNode:
if callable(func): if callable(func):
func(self) func(self)
def append_whitespaces(self, token: Token) -> None:
if self.whitespaces is None:
self.whitespaces = WhitespaceNode(token)
else:
self.whitespaces.append(token)
@dataclass(unsafe_hash=True)
class WhitespaceNode(BaseNode):
value: str
def __init__(self, token: Token[str]):
super().__init__(token.lineno, token.colno, token.filename)
self.value = ''
self.append(token)
def append(self, token: Token[str]) -> None:
self.value += token.value
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ElementaryNode(T.Generic[TV_TokenTypes], BaseNode): class ElementaryNode(T.Generic[TV_TokenTypes], BaseNode):
@ -282,23 +291,44 @@ class BooleanNode(ElementaryNode[bool]):
pass pass
class IdNode(ElementaryNode[str]): class IdNode(ElementaryNode[str]):
def __str__(self) -> str: pass
return "Id node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno)
@dataclass(unsafe_hash=True)
class NumberNode(ElementaryNode[int]): class NumberNode(ElementaryNode[int]):
raw_value: str = field(hash=False)
def __init__(self, token: Token[str]):
BaseNode.__init__(self, token.lineno, token.colno, token.filename)
self.raw_value = token.value
self.value = int(token.value, base=0)
self.bytespan = token.bytespan
class BaseStringNode(ElementaryNode[str]):
pass
@dataclass(unsafe_hash=True)
class StringNode(BaseStringNode):
raw_value: str = field(hash=False)
def __init__(self, token: Token[str], escape: bool = True):
super().__init__(token)
self.value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, token.value) if escape else token.value
self.raw_value = token.value
class FormatStringNode(StringNode):
pass pass
class StringNode(ElementaryNode[str]): @dataclass(unsafe_hash=True)
def __str__(self) -> str: class MultilineStringNode(BaseStringNode):
return "String node: '%s' (%d, %d)." % (self.value, self.lineno, self.colno)
class FormatStringNode(ElementaryNode[str]): def __init__(self, token: Token[str]):
def __str__(self) -> str: super().__init__(token)
return f"Format string node: '{self.value}' ({self.lineno}, {self.colno})." self.value = token.value
class MultilineFormatStringNode(FormatStringNode): class MultilineFormatStringNode(MultilineStringNode):
def __str__(self) -> str: pass
return f"Multiline Format string node: '{self.value}' ({self.lineno}, {self.colno})."
class ContinueNode(ElementaryNode): class ContinueNode(ElementaryNode):
pass pass
@ -306,17 +336,22 @@ class ContinueNode(ElementaryNode):
class BreakNode(ElementaryNode): class BreakNode(ElementaryNode):
pass pass
class SymbolNode(ElementaryNode[str]):
pass
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ArgumentNode(BaseNode): class ArgumentNode(BaseNode):
arguments: T.List[BaseNode] = field(hash=False) arguments: T.List[BaseNode] = field(hash=False)
commas: T.List[Token] = field(hash=False) commas: T.List[SymbolNode] = field(hash=False)
columns: T.List[SymbolNode] = field(hash=False)
kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False) kwargs: T.Dict[BaseNode, BaseNode] = field(hash=False)
def __init__(self, token: Token[TV_TokenTypes]): def __init__(self, token: Token[TV_TokenTypes]):
super().__init__(token.lineno, token.colno, token.filename) super().__init__(token.lineno, token.colno, token.filename)
self.arguments = [] self.arguments = []
self.commas = [] self.commas = []
self.columns = []
self.kwargs = {} self.kwargs = {}
self.order_error = False self.order_error = False
@ -356,227 +391,277 @@ class ArgumentNode(BaseNode):
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ArrayNode(BaseNode): class ArrayNode(BaseNode):
lbracket: SymbolNode
args: ArgumentNode args: ArgumentNode
rbracket: SymbolNode
def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): def __init__(self, lbracket: SymbolNode, args: ArgumentNode, rbracket: SymbolNode):
super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) super().__init__(lbracket.lineno, lbracket.colno, args.filename, end_lineno=rbracket.lineno, end_colno=rbracket.colno+1)
self.lbracket = lbracket
self.args = args self.args = args
self.rbracket = rbracket
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class DictNode(BaseNode): class DictNode(BaseNode):
lcurl: SymbolNode
args: ArgumentNode args: ArgumentNode
rcurl: SymbolNode
def __init__(self, args: ArgumentNode, lineno: int, colno: int, end_lineno: int, end_colno: int): def __init__(self, lcurl: SymbolNode, args: ArgumentNode, rcurl: SymbolNode):
super().__init__(lineno, colno, args.filename, end_lineno=end_lineno, end_colno=end_colno) super().__init__(lcurl.lineno, lcurl.colno, args.filename, end_lineno=rcurl.lineno, end_colno=rcurl.colno+1)
self.lcurl = lcurl
self.args = args self.args = args
self.rcurl = rcurl
class EmptyNode(BaseNode): class EmptyNode(BaseNode):
pass pass
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class OrNode(BaseNode): class BinaryOperatorNode(BaseNode):
left: BaseNode left: BaseNode
operator: SymbolNode
right: BaseNode right: BaseNode
def __init__(self, left: BaseNode, right: BaseNode): def __init__(self, left: BaseNode, operator: SymbolNode, right: BaseNode):
super().__init__(left.lineno, left.colno, left.filename) super().__init__(left.lineno, left.colno, left.filename)
self.left = left self.left = left
self.operator = operator
self.right = right self.right = right
@dataclass(unsafe_hash=True) class OrNode(BinaryOperatorNode):
class AndNode(BaseNode): pass
left: BaseNode
right: BaseNode
def __init__(self, left: BaseNode, right: BaseNode): class AndNode(BinaryOperatorNode):
super().__init__(left.lineno, left.colno, left.filename) pass
self.left = left
self.right = right
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ComparisonNode(BaseNode): class ComparisonNode(BinaryOperatorNode):
left: BaseNode
right: BaseNode
ctype: COMPARISONS ctype: COMPARISONS
def __init__(self, ctype: COMPARISONS, left: BaseNode, right: BaseNode): def __init__(self, ctype: COMPARISONS, left: BaseNode, operator: SymbolNode, right: BaseNode):
super().__init__(left.lineno, left.colno, left.filename) super().__init__(left, operator, right)
self.left = left
self.right = right
self.ctype = ctype self.ctype = ctype
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ArithmeticNode(BaseNode): class ArithmeticNode(BinaryOperatorNode):
left: BaseNode
right: BaseNode
# TODO: use a Literal for operation # TODO: use a Literal for operation
operation: str operation: str
def __init__(self, operation: str, left: BaseNode, right: BaseNode): def __init__(self, operation: str, left: BaseNode, operator: SymbolNode, right: BaseNode):
super().__init__(left.lineno, left.colno, left.filename) super().__init__(left, operator, right)
self.left = left
self.right = right
self.operation = operation self.operation = operation
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class NotNode(BaseNode): class UnaryOperatorNode(BaseNode):
operator: SymbolNode
value: BaseNode value: BaseNode
def __init__(self, token: Token[TV_TokenTypes], value: BaseNode): def __init__(self, token: Token[TV_TokenTypes], operator: SymbolNode, value: BaseNode):
super().__init__(token.lineno, token.colno, token.filename) super().__init__(token.lineno, token.colno, token.filename)
self.operator = operator
self.value = value self.value = value
class NotNode(UnaryOperatorNode):
pass
class UMinusNode(UnaryOperatorNode):
pass
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class CodeBlockNode(BaseNode): class CodeBlockNode(BaseNode):
pre_whitespaces: T.Optional[WhitespaceNode] = field(hash=False)
lines: T.List[BaseNode] = field(hash=False) lines: T.List[BaseNode] = field(hash=False)
def __init__(self, token: Token[TV_TokenTypes]): def __init__(self, token: Token[TV_TokenTypes]):
super().__init__(token.lineno, token.colno, token.filename) super().__init__(token.lineno, token.colno, token.filename)
self.pre_whitespaces = None
self.lines = [] self.lines = []
def append_whitespaces(self, token: Token) -> None:
if self.lines:
self.lines[-1].append_whitespaces(token)
elif self.pre_whitespaces is None:
self.pre_whitespaces = WhitespaceNode(token)
else:
self.pre_whitespaces.append(token)
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class IndexNode(BaseNode): class IndexNode(BaseNode):
iobject: BaseNode iobject: BaseNode
lbracket: SymbolNode
index: BaseNode index: BaseNode
rbracket: SymbolNode
def __init__(self, iobject: BaseNode, index: BaseNode): def __init__(self, iobject: BaseNode, lbracket: SymbolNode, index: BaseNode, rbracket: SymbolNode):
super().__init__(iobject.lineno, iobject.colno, iobject.filename) super().__init__(iobject.lineno, iobject.colno, iobject.filename)
self.iobject = iobject self.iobject = iobject
self.lbracket = lbracket
self.index = index self.index = index
self.rbracket = rbracket
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class MethodNode(BaseNode): class MethodNode(BaseNode):
source_object: BaseNode source_object: BaseNode
name: str dot: SymbolNode
name: IdNode
lpar: SymbolNode
args: ArgumentNode args: ArgumentNode
rpar: SymbolNode
def __init__(self, filename: str, lineno: int, colno: int, source_object: BaseNode, name: str, args: ArgumentNode): def __init__(self, source_object: BaseNode, dot: SymbolNode, name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode):
super().__init__(lineno, colno, filename) super().__init__(name.lineno, name.colno, name.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1)
self.source_object = source_object self.source_object = source_object
self.dot = dot
self.name = name self.name = name
assert isinstance(self.name, str) self.lpar = lpar
self.args = args self.args = args
self.rpar = rpar
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class FunctionNode(BaseNode): class FunctionNode(BaseNode):
func_name: str func_name: IdNode
lpar: SymbolNode
args: ArgumentNode args: ArgumentNode
rpar: SymbolNode
def __init__(self, filename: str, lineno: int, colno: int, end_lineno: int, end_colno: int, func_name: str, args: ArgumentNode): def __init__(self, func_name: IdNode, lpar: SymbolNode, args: ArgumentNode, rpar: SymbolNode):
super().__init__(lineno, colno, filename, end_lineno=end_lineno, end_colno=end_colno) super().__init__(func_name.lineno, func_name.colno, func_name.filename, end_lineno=rpar.end_lineno, end_colno=rpar.end_colno+1)
self.func_name = func_name self.func_name = func_name
assert isinstance(func_name, str) self.lpar = lpar
self.args = args self.args = args
self.rpar = rpar
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class AssignmentNode(BaseNode): class AssignmentNode(BaseNode):
var_name: str var_name: IdNode
value: BaseNode operator: SymbolNode
def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode):
super().__init__(lineno, colno, filename)
self.var_name = var_name
assert isinstance(var_name, str)
self.value = value
@dataclass(unsafe_hash=True)
class PlusAssignmentNode(BaseNode):
var_name: str
value: BaseNode value: BaseNode
def __init__(self, filename: str, lineno: int, colno: int, var_name: str, value: BaseNode): def __init__(self, var_name: IdNode, operator: SymbolNode, value: BaseNode):
super().__init__(lineno, colno, filename) super().__init__(var_name.lineno, var_name.colno, var_name.filename)
self.var_name = var_name self.var_name = var_name
assert isinstance(var_name, str) self.operator = operator
self.value = value self.value = value
class PlusAssignmentNode(AssignmentNode):
pass
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class ForeachClauseNode(BaseNode): class ForeachClauseNode(BaseNode):
varnames: T.List[str] = field(hash=False) foreach_: SymbolNode = field(hash=False)
varnames: T.List[IdNode] = field(hash=False)
commas: T.List[SymbolNode] = field(hash=False)
column: SymbolNode = field(hash=False)
items: BaseNode items: BaseNode
block: CodeBlockNode block: CodeBlockNode
endforeach: SymbolNode = field(hash=False)
def __init__(self, token: Token, varnames: T.List[str], items: BaseNode, block: CodeBlockNode): def __init__(self, foreach_: SymbolNode, varnames: T.List[IdNode], commas: T.List[SymbolNode], column: SymbolNode, items: BaseNode, block: CodeBlockNode, endforeach: SymbolNode):
super().__init__(token.lineno, token.colno, token.filename) super().__init__(foreach_.lineno, foreach_.colno, foreach_.filename)
self.foreach_ = foreach_
self.varnames = varnames self.varnames = varnames
self.commas = commas
self.column = column
self.items = items self.items = items
self.block = block self.block = block
self.endforeach = endforeach
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class IfNode(BaseNode): class IfNode(BaseNode):
if_: SymbolNode
condition: BaseNode condition: BaseNode
block: CodeBlockNode block: CodeBlockNode
def __init__(self, linenode: BaseNode, condition: BaseNode, block: CodeBlockNode): def __init__(self, linenode: BaseNode, if_node: SymbolNode, condition: BaseNode, block: CodeBlockNode):
super().__init__(linenode.lineno, linenode.colno, linenode.filename) super().__init__(linenode.lineno, linenode.colno, linenode.filename)
self.if_ = if_node
self.condition = condition self.condition = condition
self.block = block self.block = block
@dataclass(unsafe_hash=True)
class ElseNode(BaseNode):
else_: SymbolNode
block: CodeBlockNode
def __init__(self, else_: SymbolNode, block: CodeBlockNode):
super().__init__(block.lineno, block.colno, block.filename)
self.else_ = else_
self.block = block
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class IfClauseNode(BaseNode): class IfClauseNode(BaseNode):
ifs: T.List[IfNode] = field(hash=False) ifs: T.List[IfNode] = field(hash=False)
elseblock: T.Union[EmptyNode, CodeBlockNode] elseblock: T.Union[EmptyNode, ElseNode]
endif: SymbolNode
def __init__(self, linenode: BaseNode): def __init__(self, linenode: BaseNode):
super().__init__(linenode.lineno, linenode.colno, linenode.filename) super().__init__(linenode.lineno, linenode.colno, linenode.filename)
self.ifs = [] self.ifs = []
self.elseblock = None self.elseblock = EmptyNode(linenode.lineno, linenode.colno, linenode.filename)
self.endif = None
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class TestCaseClauseNode(BaseNode): class TestCaseClauseNode(BaseNode):
testcase: SymbolNode
condition: BaseNode condition: BaseNode
block: CodeBlockNode block: CodeBlockNode
endtestcase: SymbolNode
def __init__(self, condition: BaseNode, block: CodeBlockNode): def __init__(self, testcase: SymbolNode, condition: BaseNode, block: CodeBlockNode, endtestcase: SymbolNode):
super().__init__(condition.lineno, condition.colno, condition.filename) super().__init__(condition.lineno, condition.colno, condition.filename)
self.testcase = testcase
self.condition = condition self.condition = condition
self.block = block self.block = block
self.endtestcase = endtestcase
@dataclass(unsafe_hash=True)
class UMinusNode(BaseNode):
value: BaseNode
def __init__(self, current_location: Token, value: BaseNode):
super().__init__(current_location.lineno, current_location.colno, current_location.filename)
self.value = value
@dataclass(unsafe_hash=True) @dataclass(unsafe_hash=True)
class TernaryNode(BaseNode): class TernaryNode(BaseNode):
condition: BaseNode condition: BaseNode
questionmark: SymbolNode
trueblock: BaseNode trueblock: BaseNode
column: SymbolNode
falseblock: BaseNode falseblock: BaseNode
def __init__(self, condition: BaseNode, trueblock: BaseNode, falseblock: BaseNode): def __init__(self, condition: BaseNode, questionmark: SymbolNode, trueblock: BaseNode, column: SymbolNode, falseblock: BaseNode):
super().__init__(condition.lineno, condition.colno, condition.filename) super().__init__(condition.lineno, condition.colno, condition.filename)
self.condition = condition self.condition = condition
self.questionmark = questionmark
self.trueblock = trueblock self.trueblock = trueblock
self.column = column
self.falseblock = falseblock self.falseblock = falseblock
@dataclass(unsafe_hash=True)
class ParenthesizedNode(BaseNode):
lpar: SymbolNode = field(hash=False)
inner: BaseNode
rpar: SymbolNode = field(hash=False)
def __init__(self, lpar: SymbolNode, inner: BaseNode, rpar: SymbolNode):
super().__init__(lpar.lineno, lpar.colno, inner.filename, end_lineno=rpar.lineno, end_colno=rpar.colno+1)
self.lpar = lpar
self.inner = inner
self.rpar = rpar
if T.TYPE_CHECKING: if T.TYPE_CHECKING:
COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin'] COMPARISONS = Literal['==', '!=', '<', '<=', '>=', '>', 'in', 'notin']
@ -611,12 +696,30 @@ class Parser:
self.lexer = Lexer(code) self.lexer = Lexer(code)
self.stream = self.lexer.lex(filename) self.stream = self.lexer.lex(filename)
self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None) self.current: Token = Token('eof', '', 0, 0, 0, (0, 0), None)
self.previous = self.current
self.current_ws: T.List[Token] = []
self.getsym() self.getsym()
self.in_ternary = False self.in_ternary = False
def create_node(self, node_type: T.Type[BaseNodeT], *args: T.Any, **kwargs: T.Any) -> BaseNodeT:
node = node_type(*args, **kwargs)
for ws_token in self.current_ws:
node.append_whitespaces(ws_token)
self.current_ws = []
return node
def getsym(self) -> None: def getsym(self) -> None:
self.previous = self.current
try: try:
self.current = next(self.stream) self.current = next(self.stream)
while self.current.tid in {'eol', 'comment', 'whitespace'}:
self.current_ws.append(self.current)
if self.current.tid == 'eol':
break
self.current = next(self.stream)
except StopIteration: except StopIteration:
self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None) self.current = Token('eof', '', self.current.line_start, self.current.lineno, self.current.colno + self.current.bytespan[1] - self.current.bytespan[0], (0, 0), None)
@ -661,55 +764,75 @@ class Parser:
def e1(self) -> BaseNode: def e1(self) -> BaseNode:
left = self.e2() left = self.e2()
if self.accept('plusassign'): if self.accept('plusassign'):
operator = self.create_node(SymbolNode, self.previous)
value = self.e1() value = self.e1()
if not isinstance(left, IdNode): if not isinstance(left, IdNode):
raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno) raise ParseException('Plusassignment target must be an id.', self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str) assert isinstance(left.value, str)
return PlusAssignmentNode(left.filename, left.lineno, left.colno, left.value, value) return self.create_node(PlusAssignmentNode, left, operator, value)
elif self.accept('assign'): elif self.accept('assign'):
operator = self.create_node(SymbolNode, self.previous)
value = self.e1() value = self.e1()
if not isinstance(left, IdNode): if not isinstance(left, IdNode):
raise ParseException('Assignment target must be an id.', raise ParseException('Assignment target must be an id.',
self.getline(), left.lineno, left.colno) self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str) assert isinstance(left.value, str)
return AssignmentNode(left.filename, left.lineno, left.colno, left.value, value) return self.create_node(AssignmentNode, left, operator, value)
elif self.accept('questionmark'): elif self.accept('questionmark'):
if self.in_ternary: if self.in_ternary:
raise ParseException('Nested ternary operators are not allowed.', raise ParseException('Nested ternary operators are not allowed.',
self.getline(), left.lineno, left.colno) self.getline(), left.lineno, left.colno)
qm_node = self.create_node(SymbolNode, self.previous)
self.in_ternary = True self.in_ternary = True
trueblock = self.e1() trueblock = self.e1()
self.expect('colon') self.expect('colon')
column_node = self.create_node(SymbolNode, self.previous)
falseblock = self.e1() falseblock = self.e1()
self.in_ternary = False self.in_ternary = False
return TernaryNode(left, trueblock, falseblock) return self.create_node(TernaryNode, left, qm_node, trueblock, column_node, falseblock)
return left return left
def e2(self) -> BaseNode: def e2(self) -> BaseNode:
left = self.e3() left = self.e3()
while self.accept('or'): while self.accept('or'):
operator = self.create_node(SymbolNode, self.previous)
if isinstance(left, EmptyNode): if isinstance(left, EmptyNode):
raise ParseException('Invalid or clause.', raise ParseException('Invalid or clause.',
self.getline(), left.lineno, left.colno) self.getline(), left.lineno, left.colno)
left = OrNode(left, self.e3()) left = self.create_node(OrNode, left, operator, self.e3())
return left return left
def e3(self) -> BaseNode: def e3(self) -> BaseNode:
left = self.e4() left = self.e4()
while self.accept('and'): while self.accept('and'):
operator = self.create_node(SymbolNode, self.previous)
if isinstance(left, EmptyNode): if isinstance(left, EmptyNode):
raise ParseException('Invalid and clause.', raise ParseException('Invalid and clause.',
self.getline(), left.lineno, left.colno) self.getline(), left.lineno, left.colno)
left = AndNode(left, self.e4()) left = self.create_node(AndNode, left, operator, self.e4())
return left return left
def e4(self) -> BaseNode: def e4(self) -> BaseNode:
left = self.e5() left = self.e5()
for nodename, operator_type in comparison_map.items(): for nodename, operator_type in comparison_map.items():
if self.accept(nodename): if self.accept(nodename):
return ComparisonNode(operator_type, left, self.e5()) operator = self.create_node(SymbolNode, self.previous)
if self.accept('not') and self.accept('in'): return self.create_node(ComparisonNode, operator_type, left, operator, self.e5())
return ComparisonNode('notin', left, self.e5()) if self.accept('not'):
ws = self.current_ws.copy()
not_token = self.previous
if self.accept('in'):
in_token = self.previous
self.current_ws = self.current_ws[len(ws):] # remove whitespaces between not and in
temp_node = EmptyNode(in_token.lineno, in_token.colno, in_token.filename)
for w in ws:
temp_node.append_whitespaces(w)
not_token.bytespan = (not_token.bytespan[0], in_token.bytespan[1])
not_token.value += temp_node.whitespaces.value + in_token.value
operator = self.create_node(SymbolNode, not_token)
return self.create_node(ComparisonNode, 'notin', left, operator, self.e5())
return left return left
def e5(self) -> BaseNode: def e5(self) -> BaseNode:
@ -724,7 +847,8 @@ class Parser:
while True: while True:
op = self.accept_any(tuple(op_map.keys())) op = self.accept_any(tuple(op_map.keys()))
if op: if op:
left = ArithmeticNode(op_map[op], left, self.e5muldiv()) operator = self.create_node(SymbolNode, self.previous)
left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e5muldiv())
else: else:
break break
return left return left
@ -739,29 +863,34 @@ class Parser:
while True: while True:
op = self.accept_any(tuple(op_map.keys())) op = self.accept_any(tuple(op_map.keys()))
if op: if op:
left = ArithmeticNode(op_map[op], left, self.e6()) operator = self.create_node(SymbolNode, self.previous)
left = self.create_node(ArithmeticNode, op_map[op], left, operator, self.e6())
else: else:
break break
return left return left
def e6(self) -> BaseNode: def e6(self) -> BaseNode:
if self.accept('not'): if self.accept('not'):
return NotNode(self.current, self.e7()) operator = self.create_node(SymbolNode, self.previous)
return self.create_node(NotNode, self.current, operator, self.e7())
if self.accept('dash'): if self.accept('dash'):
return UMinusNode(self.current, self.e7()) operator = self.create_node(SymbolNode, self.previous)
return self.create_node(UMinusNode, self.current, operator, self.e7())
return self.e7() return self.e7()
def e7(self) -> BaseNode: def e7(self) -> BaseNode:
left = self.e8() left = self.e8()
block_start = self.current block_start = self.current
if self.accept('lparen'): if self.accept('lparen'):
lpar = self.create_node(SymbolNode, block_start)
args = self.args() args = self.args()
self.block_expect('rparen', block_start) self.block_expect('rparen', block_start)
rpar = self.create_node(SymbolNode, self.previous)
if not isinstance(left, IdNode): if not isinstance(left, IdNode):
raise ParseException('Function call must be applied to plain id', raise ParseException('Function call must be applied to plain id',
self.getline(), left.lineno, left.colno) self.getline(), left.lineno, left.colno)
assert isinstance(left.value, str) assert isinstance(left.value, str)
left = FunctionNode(left.filename, left.lineno, left.colno, self.current.lineno, self.current.colno, left.value, args) left = self.create_node(FunctionNode, left, lpar, args, rpar)
go_again = True go_again = True
while go_again: while go_again:
go_again = False go_again = False
@ -776,17 +905,23 @@ class Parser:
def e8(self) -> BaseNode: def e8(self) -> BaseNode:
block_start = self.current block_start = self.current
if self.accept('lparen'): if self.accept('lparen'):
lpar = self.create_node(SymbolNode, block_start)
e = self.statement() e = self.statement()
self.block_expect('rparen', block_start) self.block_expect('rparen', block_start)
return e rpar = self.create_node(SymbolNode, self.previous)
return ParenthesizedNode(lpar, e, rpar)
elif self.accept('lbracket'): elif self.accept('lbracket'):
lbracket = self.create_node(SymbolNode, block_start)
args = self.args() args = self.args()
self.block_expect('rbracket', block_start) self.block_expect('rbracket', block_start)
return ArrayNode(args, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) rbracket = self.create_node(SymbolNode, self.previous)
return self.create_node(ArrayNode, lbracket, args, rbracket)
elif self.accept('lcurl'): elif self.accept('lcurl'):
lcurl = self.create_node(SymbolNode, block_start)
key_values = self.key_values() key_values = self.key_values()
self.block_expect('rcurl', block_start) self.block_expect('rcurl', block_start)
return DictNode(key_values, block_start.lineno, block_start.colno, self.current.lineno, self.current.colno) rcurl = self.create_node(SymbolNode, self.previous)
return self.create_node(DictNode, lcurl, key_values, rcurl)
else: else:
return self.e9() return self.e9()
@ -794,33 +929,35 @@ class Parser:
t = self.current t = self.current
if self.accept('true'): if self.accept('true'):
t.value = True t.value = True
return BooleanNode(t) return self.create_node(BooleanNode, t)
if self.accept('false'): if self.accept('false'):
t.value = False t.value = False
return BooleanNode(t) return self.create_node(BooleanNode, t)
if self.accept('id'): if self.accept('id'):
return IdNode(t) return self.create_node(IdNode, t)
if self.accept('number'): if self.accept('number'):
return NumberNode(t) return self.create_node(NumberNode, t)
if self.accept('string'): if self.accept('string'):
return StringNode(t) return self.create_node(StringNode, t)
if self.accept('fstring'): if self.accept('fstring'):
return FormatStringNode(t) return self.create_node(FormatStringNode, t)
if self.accept('multiline_string'):
return self.create_node(MultilineStringNode, t)
if self.accept('multiline_fstring'): if self.accept('multiline_fstring'):
return MultilineFormatStringNode(t) return self.create_node(MultilineFormatStringNode, t)
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
def key_values(self) -> ArgumentNode: def key_values(self) -> ArgumentNode:
s = self.statement() s = self.statement()
a = ArgumentNode(self.current) a = self.create_node(ArgumentNode, self.current)
while not isinstance(s, EmptyNode): while not isinstance(s, EmptyNode):
if self.accept('colon'): if self.accept('colon'):
a.columns.append(self.create_node(SymbolNode, self.previous))
a.set_kwarg_no_check(s, self.statement()) a.set_kwarg_no_check(s, self.statement())
potential = self.current
if not self.accept('comma'): if not self.accept('comma'):
return a return a
a.commas.append(potential) a.commas.append(self.create_node(SymbolNode, self.previous))
else: else:
raise ParseException('Only key:value pairs are valid in dict construction.', raise ParseException('Only key:value pairs are valid in dict construction.',
self.getline(), s.lineno, s.colno) self.getline(), s.lineno, s.colno)
@ -829,22 +966,21 @@ class Parser:
def args(self) -> ArgumentNode: def args(self) -> ArgumentNode:
s = self.statement() s = self.statement()
a = ArgumentNode(self.current) a = self.create_node(ArgumentNode, self.current)
while not isinstance(s, EmptyNode): while not isinstance(s, EmptyNode):
potential = self.current
if self.accept('comma'): if self.accept('comma'):
a.commas.append(potential) a.commas.append(self.create_node(SymbolNode, self.previous))
a.append(s) a.append(s)
elif self.accept('colon'): elif self.accept('colon'):
a.columns.append(self.create_node(SymbolNode, self.previous))
if not isinstance(s, IdNode): if not isinstance(s, IdNode):
raise ParseException('Dictionary key must be a plain identifier.', raise ParseException('Dictionary key must be a plain identifier.',
self.getline(), s.lineno, s.colno) self.getline(), s.lineno, s.colno)
a.set_kwarg(s, self.statement()) a.set_kwarg(s, self.statement())
potential = self.current
if not self.accept('comma'): if not self.accept('comma'):
return a return a
a.commas.append(potential) a.commas.append(self.create_node(SymbolNode, self.previous))
else: else:
a.append(s) a.append(s)
return a return a
@ -852,70 +988,87 @@ class Parser:
return a return a
def method_call(self, source_object: BaseNode) -> MethodNode: def method_call(self, source_object: BaseNode) -> MethodNode:
dot = self.create_node(SymbolNode, self.previous)
methodname = self.e9() methodname = self.e9()
if not isinstance(methodname, IdNode): if not isinstance(methodname, IdNode):
if isinstance(source_object, NumberNode) and isinstance(methodname, NumberNode):
raise ParseException('meson does not support float numbers',
self.getline(), source_object.lineno, source_object.colno)
raise ParseException('Method name must be plain id', raise ParseException('Method name must be plain id',
self.getline(), self.current.lineno, self.current.colno) self.getline(), self.current.lineno, self.current.colno)
assert isinstance(methodname.value, str) assert isinstance(methodname.value, str)
self.expect('lparen') self.expect('lparen')
lpar = self.create_node(SymbolNode, self.previous)
args = self.args() args = self.args()
rpar = self.create_node(SymbolNode, self.current)
self.expect('rparen') self.expect('rparen')
method = MethodNode(methodname.filename, methodname.lineno, methodname.colno, source_object, methodname.value, args) method = self.create_node(MethodNode, source_object, dot, methodname, lpar, args, rpar)
if self.accept('dot'): if self.accept('dot'):
return self.method_call(method) return self.method_call(method)
return method return method
def index_call(self, source_object: BaseNode) -> IndexNode: def index_call(self, source_object: BaseNode) -> IndexNode:
lbracket = self.create_node(SymbolNode, self.previous)
index_statement = self.statement() index_statement = self.statement()
self.expect('rbracket') self.expect('rbracket')
return IndexNode(source_object, index_statement) rbracket = self.create_node(SymbolNode, self.previous)
return self.create_node(IndexNode, source_object, lbracket, index_statement, rbracket)
def foreachblock(self) -> ForeachClauseNode: def foreachblock(self) -> ForeachClauseNode:
t = self.current foreach_ = self.create_node(SymbolNode, self.previous)
self.expect('id') self.expect('id')
assert isinstance(t.value, str) assert isinstance(self.previous.value, str)
varname = t varnames = [self.create_node(IdNode, self.previous)]
varnames = [t.value] commas = []
if self.accept('comma'): if self.accept('comma'):
t = self.current commas.append(self.create_node(SymbolNode, self.previous))
self.expect('id') self.expect('id')
assert isinstance(t.value, str) assert isinstance(self.previous.value, str)
varnames.append(t.value) varnames.append(self.create_node(IdNode, self.previous))
self.expect('colon') self.expect('colon')
column = self.create_node(SymbolNode, self.previous)
items = self.statement() items = self.statement()
block = self.codeblock() block = self.codeblock()
return ForeachClauseNode(varname, varnames, items, block) endforeach = self.create_node(SymbolNode, self.current)
return self.create_node(ForeachClauseNode, foreach_, varnames, commas, column, items, block, endforeach)
def ifblock(self) -> IfClauseNode: def ifblock(self) -> IfClauseNode:
if_node = self.create_node(SymbolNode, self.previous)
condition = self.statement() condition = self.statement()
clause = IfClauseNode(condition) clause = self.create_node(IfClauseNode, condition)
self.expect('eol') self.expect('eol')
block = self.codeblock() block = self.codeblock()
clause.ifs.append(IfNode(clause, condition, block)) clause.ifs.append(self.create_node(IfNode, clause, if_node, condition, block))
self.elseifblock(clause) self.elseifblock(clause)
clause.elseblock = self.elseblock() clause.elseblock = self.elseblock()
clause.endif = self.create_node(SymbolNode, self.current)
return clause return clause
def elseifblock(self, clause: IfClauseNode) -> None: def elseifblock(self, clause: IfClauseNode) -> None:
while self.accept('elif'): while self.accept('elif'):
elif_ = self.create_node(SymbolNode, self.previous)
s = self.statement() s = self.statement()
self.expect('eol') self.expect('eol')
b = self.codeblock() b = self.codeblock()
clause.ifs.append(IfNode(s, s, b)) clause.ifs.append(self.create_node(IfNode, s, elif_, s, b))
def elseblock(self) -> T.Union[CodeBlockNode, EmptyNode]: def elseblock(self) -> T.Union[ElseNode, EmptyNode]:
if self.accept('else'): if self.accept('else'):
else_ = self.create_node(SymbolNode, self.previous)
self.expect('eol') self.expect('eol')
return self.codeblock() block = self.codeblock()
return ElseNode(else_, block)
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename) return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
def testcaseblock(self) -> TestCaseClauseNode: def testcaseblock(self) -> TestCaseClauseNode:
testcase = self.create_node(SymbolNode, self.previous)
condition = self.statement() condition = self.statement()
self.expect('eol') self.expect('eol')
block = self.codeblock() block = self.codeblock()
return TestCaseClauseNode(condition, block) endtestcase = SymbolNode(self.current)
return self.create_node(TestCaseClauseNode, testcase, condition, block, endtestcase)
def line(self) -> BaseNode: def line(self) -> BaseNode:
block_start = self.current block_start = self.current
@ -930,9 +1083,9 @@ class Parser:
self.block_expect('endforeach', block_start) self.block_expect('endforeach', block_start)
return forblock return forblock
if self.accept('continue'): if self.accept('continue'):
return ContinueNode(self.current) return self.create_node(ContinueNode, self.current)
if self.accept('break'): if self.accept('break'):
return BreakNode(self.current) return self.create_node(BreakNode, self.current)
if self.lexer.in_unit_test and self.accept('testcase'): if self.lexer.in_unit_test and self.accept('testcase'):
block = self.testcaseblock() block = self.testcaseblock()
self.block_expect('endtestcase', block_start) self.block_expect('endtestcase', block_start)
@ -940,15 +1093,29 @@ class Parser:
return self.statement() return self.statement()
def codeblock(self) -> CodeBlockNode: def codeblock(self) -> CodeBlockNode:
block = CodeBlockNode(self.current) block = self.create_node(CodeBlockNode, self.current)
cond = True cond = True
try: try:
while cond: while cond:
for ws_token in self.current_ws:
block.append_whitespaces(ws_token)
self.current_ws = []
curline = self.line() curline = self.line()
if not isinstance(curline, EmptyNode): if not isinstance(curline, EmptyNode):
block.lines.append(curline) block.lines.append(curline)
cond = self.accept('eol') cond = self.accept('eol')
except ParseException as e: except ParseException as e:
e.ast = block e.ast = block
raise raise
# Remaining whitespaces will not be catched since there are no more nodes
for ws_token in self.current_ws:
block.append_whitespaces(ws_token)
self.current_ws = []
return block return block

8
mesonbuild/optinterpreter.py
Unescape View File

@ -113,7 +113,9 @@ class OptionInterpreter:
def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var': def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var':
if isinstance(arg, str): if isinstance(arg, str):
return arg return arg
elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, if isinstance(arg, mparser.ParenthesizedNode):
return self.reduce_single(arg.inner)
elif isinstance(arg, (mparser.BaseStringNode, mparser.BooleanNode,
mparser.NumberNode)): mparser.NumberNode)):
return arg.value return arg.value
elif isinstance(arg, mparser.ArrayNode): elif isinstance(arg, mparser.ArrayNode):
@ -121,7 +123,7 @@ class OptionInterpreter:
elif isinstance(arg, mparser.DictNode): elif isinstance(arg, mparser.DictNode):
d = {} d = {}
for k, v in arg.args.kwargs.items(): for k, v in arg.args.kwargs.items():
if not isinstance(k, mparser.StringNode): if not isinstance(k, mparser.BaseStringNode):
raise OptionException('Dictionary keys must be a string literal') raise OptionException('Dictionary keys must be a string literal')
d[k.value] = self.reduce_single(v) d[k.value] = self.reduce_single(v)
return d return d
@ -162,7 +164,7 @@ class OptionInterpreter:
def evaluate_statement(self, node: mparser.BaseNode) -> None: def evaluate_statement(self, node: mparser.BaseNode) -> None:
if not isinstance(node, mparser.FunctionNode): if not isinstance(node, mparser.FunctionNode):
raise OptionException('Option file may only contain option definitions') raise OptionException('Option file may only contain option definitions')
func_name = node.func_name func_name = node.func_name.value
if func_name != 'option': if func_name != 'option':
raise OptionException('Only calls to option() are allowed in option files.') raise OptionException('Only calls to option() are allowed in option files.')
(posargs, kwargs) = self.reduce_arguments(node.args) (posargs, kwargs) = self.reduce_arguments(node.args)

47
mesonbuild/rewriter.py
Unescape View File

@ -28,7 +28,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL
from mesonbuild.mesonlib import MesonException, setup_vsenv from mesonbuild.mesonlib import MesonException, setup_vsenv
from . import mlog, environment from . import mlog, environment
from functools import wraps from functools import wraps
from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode
import json, os, re, sys import json, os, re, sys
import typing as T import typing as T
@ -104,6 +104,9 @@ class RequiredKeys:
return wrapped return wrapped
def _symbol(val: str) -> SymbolNode:
return SymbolNode(Token('', '', 0, 0, 0, (0, 0), val))
class MTypeBase: class MTypeBase:
def __init__(self, node: T.Optional[BaseNode] = None): def __init__(self, node: T.Optional[BaseNode] = None):
if node is None: if node is None:
@ -189,7 +192,7 @@ class MTypeList(MTypeBase):
super().__init__(node) super().__init__(node)
def _new_node(self): def _new_node(self):
return ArrayNode(ArgumentNode(Token('', '', 0, 0, 0, None, '')), 0, 0, 0, 0) return ArrayNode(_symbol('['), ArgumentNode(Token('', '', 0, 0, 0, None, '')), _symbol(']'))
def _new_element_node(self, value): def _new_element_node(self, value):
# Overwrite in derived class # Overwrite in derived class
@ -267,12 +270,12 @@ class MTypeStrList(MTypeList):
return StringNode(Token('', '', 0, 0, 0, None, str(value))) return StringNode(Token('', '', 0, 0, 0, None, str(value)))
def _check_is_equal(self, node, value) -> bool: def _check_is_equal(self, node, value) -> bool:
if isinstance(node, StringNode): if isinstance(node, BaseStringNode):
return node.value == value return node.value == value
return False return False
def _check_regex_matches(self, node, regex: str) -> bool: def _check_regex_matches(self, node, regex: str) -> bool:
if isinstance(node, StringNode): if isinstance(node, BaseStringNode):
return re.match(regex, node.value) is not None return re.match(regex, node.value) is not None
return False return False
@ -292,7 +295,7 @@ class MTypeIDList(MTypeList):
return False return False
def _check_regex_matches(self, node, regex: str) -> bool: def _check_regex_matches(self, node, regex: str) -> bool:
if isinstance(node, StringNode): if isinstance(node, BaseStringNode):
return re.match(regex, node.value) is not None return re.match(regex, node.value) is not None
return False return False
@ -420,7 +423,7 @@ class Rewriter:
if target in self.interpreter.assignments: if target in self.interpreter.assignments:
node = self.interpreter.assignments[target] node = self.interpreter.assignments[target]
if isinstance(node, FunctionNode): if isinstance(node, FunctionNode):
if node.func_name in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}: if node.func_name.value in {'executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries'}:
tgt = self.interpreter.assign_vals[target] tgt = self.interpreter.assign_vals[target]
return tgt return tgt
@ -440,7 +443,7 @@ class Rewriter:
if dependency in self.interpreter.assignments: if dependency in self.interpreter.assignments:
node = self.interpreter.assignments[dependency] node = self.interpreter.assignments[dependency]
if isinstance(node, FunctionNode): if isinstance(node, FunctionNode):
if node.func_name == 'dependency': if node.func_name.value == 'dependency':
name = self.interpreter.flatten_args(node.args)[0] name = self.interpreter.flatten_args(node.args)[0]
dep = check_list(name) dep = check_list(name)
@ -630,7 +633,7 @@ class Rewriter:
args = [] args = []
if isinstance(n, FunctionNode): if isinstance(n, FunctionNode):
args = list(n.args.arguments) args = list(n.args.arguments)
if n.func_name in BUILD_TARGET_FUNCTIONS: if n.func_name.value in BUILD_TARGET_FUNCTIONS:
args.pop(0) args.pop(0)
elif isinstance(n, ArrayNode): elif isinstance(n, ArrayNode):
args = n.args.arguments args = n.args.arguments
@ -652,7 +655,7 @@ class Rewriter:
src_list = [] src_list = []
for i in target['sources']: for i in target['sources']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
src_list += [j.value] src_list += [j.value]
# Generate the new String nodes # Generate the new String nodes
@ -686,7 +689,7 @@ class Rewriter:
def find_node(src): def find_node(src):
for i in target['sources']: for i in target['sources']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
if j.value == src: if j.value == src:
return i, j return i, j
return None, None return None, None
@ -728,7 +731,7 @@ class Rewriter:
node = tgt_function.args.kwargs[extra_files_key] node = tgt_function.args.kwargs[extra_files_key]
except StopIteration: except StopIteration:
# Target has no extra_files kwarg, create one # Target has no extra_files kwarg, create one
node = ArrayNode(ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), tgt_function.end_lineno, tgt_function.end_colno, tgt_function.end_lineno, tgt_function.end_colno) node = ArrayNode(_symbol('['), ArgumentNode(Token('', tgt_function.filename, 0, 0, 0, None, '[]')), _symbol(']'))
tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node tgt_function.args.kwargs[IdNode(Token('string', tgt_function.filename, 0, 0, 0, None, 'extra_files'))] = node
mark_array = False mark_array = False
if tgt_function not in self.modified_nodes: if tgt_function not in self.modified_nodes:
@ -745,7 +748,7 @@ class Rewriter:
extra_files_list = [] extra_files_list = []
for i in target['extra_files']: for i in target['extra_files']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
extra_files_list += [j.value] extra_files_list += [j.value]
# Generate the new String nodes # Generate the new String nodes
@ -776,7 +779,7 @@ class Rewriter:
def find_node(src): def find_node(src):
for i in target['extra_files']: for i in target['extra_files']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
if j.value == src: if j.value == src:
return i, j return i, j
return None, None return None, None
@ -812,17 +815,17 @@ class Rewriter:
# Build src list # Build src list
src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, ''))
src_arr_node = ArrayNode(src_arg_node, 0, 0, 0, 0) src_arr_node = ArrayNode(_symbol('['), src_arg_node, _symbol(']'))
src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) src_far_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, ''))
src_fun_node = FunctionNode(filename, 0, 0, 0, 0, 'files', src_far_node) src_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), 'files')), _symbol('('), src_far_node, _symbol(')'))
src_ass_node = AssignmentNode(filename, 0, 0, source_id, src_fun_node) src_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), source_id)), _symbol('='), src_fun_node)
src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']] src_arg_node.arguments = [StringNode(Token('string', filename, 0, 0, 0, None, x)) for x in cmd['sources']]
src_far_node.arguments = [src_arr_node] src_far_node.arguments = [src_arr_node]
# Build target # Build target
tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) tgt_arg_node = ArgumentNode(Token('string', filename, 0, 0, 0, None, ''))
tgt_fun_node = FunctionNode(filename, 0, 0, 0, 0, cmd['target_type'], tgt_arg_node) tgt_fun_node = FunctionNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), cmd['target_type'])), _symbol('('), tgt_arg_node, _symbol(')'))
tgt_ass_node = AssignmentNode(filename, 0, 0, target_id, tgt_fun_node) tgt_ass_node = AssignmentNode(IdNode(Token('id', filename, 0, 0, 0, (0, 0), target_id)), _symbol('='), tgt_fun_node)
tgt_arg_node.arguments = [ tgt_arg_node.arguments = [
StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])), StringNode(Token('string', filename, 0, 0, 0, None, cmd['target'])),
IdNode(Token('string', filename, 0, 0, 0, None, source_id)) IdNode(Token('string', filename, 0, 0, 0, None, source_id))
@ -845,12 +848,12 @@ class Rewriter:
src_list = [] src_list = []
for i in target['sources']: for i in target['sources']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
src_list += [j.value] src_list += [j.value]
extra_files_list = [] extra_files_list = []
for i in target['extra_files']: for i in target['extra_files']:
for j in arg_list_from_node(i): for j in arg_list_from_node(i):
if isinstance(j, StringNode): if isinstance(j, BaseStringNode):
extra_files_list += [j.value] extra_files_list += [j.value]
test_data = { test_data = {
'name': target['name'], 'name': target['name'],
@ -865,8 +868,8 @@ class Rewriter:
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))]) path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))])
unknown = [x for x in i.arguments if not isinstance(x, StringNode)] unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)]
sources = [x for x in i.arguments if isinstance(x, StringNode)] sources = [x for x in i.arguments if isinstance(x, BaseStringNode)]
sources = sorted(sources, key=lambda x: path_sorter(x.value)) sources = sorted(sources, key=lambda x: path_sorter(x.value))
i.arguments = unknown + sources i.arguments = unknown + sources

1
run_format_tests.py
Unescape View File

@ -63,6 +63,7 @@ def check_format() -> None:
'work area', 'work area',
'.eggs', '_cache', # e.g. .mypy_cache '.eggs', '_cache', # e.g. .mypy_cache
'venv', # virtualenvs have DOS line endings 'venv', # virtualenvs have DOS line endings
'118 rewrite', # we explicitly test for tab in meson.build file
} }
for (root, _, filenames) in os.walk('.'): for (root, _, filenames) in os.walk('.'):
if any([x in root for x in skip_dirs]): if any([x in root for x in skip_dirs]):

189
test cases/unit/118 rewrite/meson.build
Unescape View File

@ -0,0 +1,189 @@
# This file should expose all possible meson syntaxes
# and ensure the AstInterpreter and RawPrinter are able
# to parse and write a file identical to the original.
project ( # project comment 1
# project comment 2
'rewrite' , # argument comment
# project comment 3
'cpp',
'c',
default_options: [
'unity=on',
'unity_size=50', # number of cpp / unity. default is 4...
'warning_level=2', # eqv to /W3
'werror=true', # treat warnings as errors
'b_ndebug=if-release', # disable assert in Release
'cpp_eh=a', # /EHa exception handling
'cpp_std=c++17',
'cpp_winlibs=' + ','.join([ # array comment
# in array
# comment
'kernel32.lib',
'user32.lib',
'gdi32.lib',
'winspool.lib',
'comdlg32.lib',
'advapi32.lib',
'shell32.lib'
# before comma comment
,
# after comma comment
'ole32.lib',
'oleaut32.lib',
'uuid.lib',
'odbc32.lib',
'odbccp32.lib',
'Delayimp.lib', # For delay loaded dll
'OLDNAMES.lib',
'dbghelp.lib',
'psapi.lib',
]),
],
meson_version: '>=1.2',
version: '1.0.0',
) # project comment 4
cppcoro_dep = dependency('andreasbuhr-cppcoro-cppcoro')
cppcoro = declare_dependency(
dependencies: [cppcoro_dep.partial_dependency(
includes: true,
link_args: true,
links: true,
sources: true,
)],
# '/await:strict' allows to use <coroutine> rather than <experimental/coroutine> with C++17.
# We can remove '/await:strict' once we update to C++20.
compile_args: ['/await:strict'],
# includes:true doesn't work for now in partial_dependency()
# This line could be removed once https://github.com/mesonbuild/meson/pull/10122 is released.
include_directories: cppcoro_dep.get_variable('includedir1'),
)
if get_option('unicode') #if comment
#if comment 2
mfc=cpp_compiler.find_library(get_option('debug')?'mfc140ud':'mfc140u')
# if comment 3
else#elsecommentnowhitespaces
# else comment 1
mfc = cpp_compiler.find_library( get_option( 'debug' ) ? 'mfc140d' : 'mfc140')
# else comment 2
endif #endif comment
assert(1 in [1, 2], '''1 should be in [1, 2]''')
assert(3 not in [1, 2], '''3 shouldn't be in [1, 2]''')
assert(not (3 in [1, 2]), '''3 shouldn't be in [1, 2]''')
assert('b' in ['a', 'b'], ''''b' should be in ['a', 'b']''')
assert('c' not in ['a', 'b'], ''''c' shouldn't be in ['a', 'b']''')
assert(exe1 in [exe1, exe2], ''''exe1 should be in [exe1, exe2]''')
assert(exe3 not in [exe1, exe2], ''''exe3 shouldn't be in [exe1, exe2]''')
assert('a' in {'a': 'b'}, '''1 should be in {'a': 'b'}''')
assert('b'not in{'a':'b'}, '''1 should be in {'a': 'b'}''')
assert('a'in'abc')
assert('b' not in 'def')
w = 'world'
d = {'a': 1, 'b': 0b10101010, 'c': 'pi', 'd': '''a
b
c''', 'e': f'hello @w@', 'f': f'''triple
formatted
string # this is not a comment
hello @w@
''', 'g': [1, 2, 3],
'h' # comment a
: # comment b
0xDEADBEEF # comment c
, # comment d
'hh': 0xfeedc0de, # lowercase hexa
'hhh': 0XaBcD0123, # mixed case hexa
'oo': 0O123456, # upper O octa
'bb': 0B1111, # upper B binary
'i': {'aa': 11, # this is a comment
'bb': 22}, # a comment inside a dict
'o': 0o754,
'm': -12, # minus number
'eq': 1 + 3 - 3 % 4 + -( 7 * 8 ),
} # end of dict comment
hw = d['e']
one = d['g'][0]
w += '!'
components = {
'foo': ['foo.c'],
'bar': ['bar.c'],
'baz': ['baz.c'], # this line is indented with a tab!
}
# compute a configuration based on system dependencies, custom logic
conf = configuration_data()
conf.set('USE_FOO', 1)
# Determine the sources to compile
sources_to_compile = []
foreach name, sources : components
if conf.get('USE_@0@'.format(name.to_upper()), 0) == 1
sources_to_compile += sources
endif
endforeach
items = ['a', 'continue', 'b', 'break', 'c']
result = []
foreach i : items
if i == 'continue'
continue
elif i == 'break'
break
endif
result += i
endforeach
# result is ['a', 'b']
if a and b
# do something
endif
if c or d
# do something
endif
if not e
# do something
endif
if not (f or g)
# do something
endif
single_quote = 'contains a \' character'
string_escapes = '\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}'
no_string_escapes = '''\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}'''
# FIXME: is it supposed to work? (cont_eol inside string)
# cont_string = 'blablabla\
# blablabla'
# cont_eol with whitespace and comments after
if a \ # comment in cont 1
and b \ # comment in cont 2
or c # comment in cont 3
message('ok')
endif
if a \
or b
debug('help!')
endif
# End of file comment with no linebreak

22
unittests/rewritetests.py
Unescape View File

@ -13,11 +13,15 @@
# limitations under the License. # limitations under the License.
import subprocess import subprocess
from itertools import zip_longest
import json import json
import os import os
from pathlib import Path
import shutil import shutil
import unittest import unittest
from mesonbuild.ast import IntrospectionInterpreter, AstIDGenerator
from mesonbuild.ast.printer import RawPrinter
from mesonbuild.mesonlib import windows_proof_rmtree from mesonbuild.mesonlib import windows_proof_rmtree
from .baseplatformtests import BasePlatformTests from .baseplatformtests import BasePlatformTests
@ -396,3 +400,21 @@ class RewriterTests(BasePlatformTests):
# Check the written file # Check the written file
out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json')) out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
self.assertDictEqual(out, expected) self.assertDictEqual(out, expected)
def test_raw_printer_is_idempotent(self):
test_path = Path(self.unit_test_dir, '118 rewrite')
meson_build_file = test_path / 'meson.build'
# original_contents = meson_build_file.read_bytes()
original_contents = meson_build_file.read_text(encoding='utf-8')
interpreter = IntrospectionInterpreter(test_path, '', 'ninja', visitors = [AstIDGenerator()])
interpreter.analyze()
printer = RawPrinter()
interpreter.ast.accept(printer)
# new_contents = printer.result.encode('utf-8')
new_contents = printer.result
# Do it line per line because it is easier to debug like that
for orig_line, new_line in zip_longest(original_contents.splitlines(), new_contents.splitlines()):
self.assertEqual(orig_line, new_line)

Write Preview
Loading…
Cancel
Save