Basic AST visitor pattern

mesonbuild/ast/__init__.py

@@ -0,0 +1,25 @@
+# Copyright 2019 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This class contains the basic functionality needed to run any interpreter
+# or an interpreter-based tool.
+
+__all__ = [
+    'AstInterpreter',
+    'RewriterInterpreter',
+    'AstVisitor'
+]
+
+from .interpreter import (AstInterpreter, RewriterInterpreter)
+from .visitor import AstVisitor

mesonbuild/ast/interpreter.py

@@ -15,10 +15,10 @@
 # This class contains the basic functionality needed to run any interpreter
 # or an interpreter-based tool.
 
-from . import interpreterbase, mparser, mesonlib
-from . import environment
+from .. import interpreterbase, mparser, mesonlib
+from .. import environment
 
-from .interpreterbase import InterpreterException, InvalidArguments, BreakRequest, ContinueRequest
+from ..interpreterbase import InterpreterException, InvalidArguments, BreakRequest, ContinueRequest
 
 import os, sys
 
@@ -46,6 +46,7 @@ REMOVE_SOURCE = 1
 class AstInterpreter(interpreterbase.InterpreterBase):
     def __init__(self, source_root, subdir):
         super().__init__(source_root, subdir)
+        self.visited_subdirs = {}
         self.funcs.update({'project': self.func_do_nothing,
                            'test': self.func_do_nothing,
                            'benchmark': self.func_do_nothing,
@@ -83,7 +84,7 @@ class AstInterpreter(interpreterbase.InterpreterBase):
                            'build_target': self.func_do_nothing,
                            'custom_target': self.func_do_nothing,
                            'run_target': self.func_do_nothing,
-                           'subdir': self.func_do_nothing,
+                           'subdir': self.func_subdir,
                            'set_variable': self.func_do_nothing,
                            'get_variable': self.func_do_nothing,
                            'is_variable': self.func_do_nothing,
@@ -92,6 +93,39 @@ class AstInterpreter(interpreterbase.InterpreterBase):
     def func_do_nothing(self, node, args, kwargs):
         return True
 
+    def func_subdir(self, node, args, kwargs):
+        args = self.flatten_args(args)
+        if len(args) != 1 or not isinstance(args[0], str):
+            sys.stderr.write('Unable to evaluate subdir({}) in AstInterpreter --> Skipping\n'.format(args))
+            return
+
+        prev_subdir = self.subdir
+        subdir = os.path.join(prev_subdir, args[0])
+        absdir = os.path.join(self.source_root, subdir)
+        buildfilename = os.path.join(self.subdir, environment.build_filename)
+        absname = os.path.join(self.source_root, buildfilename)
+        symlinkless_dir = os.path.realpath(absdir)
+        if symlinkless_dir in self.visited_subdirs:
+            sys.stderr.write('Trying to enter {} which has already been visited --> Skipping\n'.format(args[0]))
+            return
+        self.visited_subdirs[symlinkless_dir] = True
+
+        if not os.path.isfile(absname):
+            sys.stderr.write('Unable to find build file {} --> Skipping\n'.format(buildfilename))
+            return
+        with open(absname, encoding='utf8') as f:
+            code = f.read()
+        assert(isinstance(code, str))
+        try:
+            codeblock = mparser.Parser(code, self.subdir).parse()
+        except mesonlib.MesonException as me:
+            me.file = buildfilename
+            raise me
+
+        self.subdir = subdir
+        self.evaluate_codeblock(codeblock)
+        self.subdir = prev_subdir
+
     def method_call(self, node):
         return True
 
@@ -136,6 +170,20 @@ class AstInterpreter(interpreterbase.InterpreterBase):
     def assignment(self, node):
         pass
 
+    def flatten_args(self, args):
+        # Resolve mparser.ArrayNode if needed
+        flattend_args = []
+        if isinstance(args, mparser.ArrayNode):
+            args = [x.value for x in args.args.arguments]
+        for i in args:
+            if isinstance(i, mparser.ArrayNode):
+                flattend_args += [x.value for x in i.args.arguments]
+            elif isinstance(i, str):
+                flattend_args += [i]
+            else:
+                pass
+        return flattend_args
+
 class RewriterInterpreter(AstInterpreter):
     def __init__(self, source_root, subdir):
         super().__init__(source_root, subdir)
@@ -197,7 +245,10 @@ class RewriterInterpreter(AstInterpreter):
         except mesonlib.MesonException as me:
             me.file = buildfilename
             raise me
-        self.evaluate_codeblock(codeblock)
+        try:
+            self.evaluate_codeblock(codeblock)
+        except SubdirDoneRequest:
+            pass
         self.subdir = prev_subdir
 
     def func_files(self, node, args, kwargs):

mesonbuild/ast/printer.py

@@ -0,0 +1,22 @@
+# Copyright 2018 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This class contains the basic functionality needed to run any interpreter
+# or an interpreter-based tool
+
+from . import AstVisitor
+
+class AstPrinter(AstVisitor):
+    def __init__(self):
+        pass

mesonbuild/ast/visitor.py

@@ -0,0 +1,118 @@
+# Copyright 2018 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This class contains the basic functionality needed to run any interpreter
+# or an interpreter-based tool
+
+from .. import mparser
+
+class AstVisitor:
+    def __init__(self):
+        pass
+
+    def visit_BooleanNode(self, node: mparser.BooleanNode):
+        pass
+
+    def visit_IdNode(self, node: mparser.IdNode):
+        pass
+
+    def visit_NumberNode(self, node: mparser.NumberNode):
+        pass
+
+    def visit_StringNode(self, node: mparser.StringNode):
+        pass
+
+    def visit_ContinueNode(self, node: mparser.ContinueNode):
+        pass
+
+    def visit_BreakNode(self, node: mparser.BreakNode):
+        pass
+
+    def visit_ArrayNode(self, node: mparser.ArrayNode):
+        node.args.accept(self)
+
+    def visit_DictNode(self, node: mparser.DictNode):
+        node.args.accept(self)
+
+    def visit_EmptyNode(self, node: mparser.EmptyNode):
+        pass
+
+    def visit_OrNode(self, node: mparser.OrNode):
+        node.left.accept(self)
+        node.right.accept(self)
+
+    def visit_AndNode(self, node: mparser.AndNode):
+        node.left.accept(self)
+        node.right.accept(self)
+
+    def visit_ComparisonNode(self, node: mparser.ComparisonNode):
+        node.left.accept(self)
+        node.right.accept(self)
+
+    def visit_ArithmeticNode(self, node: mparser.ArithmeticNode):
+        node.left.accept(self)
+        node.right.accept(self)
+
+    def visit_NotNode(self, node: mparser.NotNode):
+        node.value.accept(self)
+
+    def visit_CodeBlockNode(self, node: mparser.CodeBlockNode):
+        for i in node.lines:
+            i.accept(self)
+
+    def visit_IndexNode(self, node: mparser.IndexNode):
+        node.index.accept(self)
+
+    def visit_MethodNode(self, node: mparser.MethodNode):
+        node.source_object.accept(self)
+        node.args.accept(self)
+
+    def visit_FunctionNode(self, node: mparser.FunctionNode):
+        node.args.accept(self)
+
+    def visit_AssignmentNode(self, node: mparser.AssignmentNode):
+        node.value.accept(self)
+
+    def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode):
+        node.value.accept(self)
+
+    def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode):
+        node.items.accept(self)
+        node.block.accept(self)
+
+    def visit_IfClauseNode(self, node: mparser.IfClauseNode):
+        for i in node.ifs:
+            i.accept(self)
+        if node.elseblock:
+            node.elseblock.accept(self)
+
+    def visit_UMinusNode(self, node: mparser.UMinusNode):
+        node.value.accept(self)
+
+    def visit_IfNode(self, node: mparser.IfNode):
+        node.condition.accept(self)
+        node.block.accept(self)
+
+    def visit_TernaryNode(self, node: mparser.TernaryNode):
+        node.condition.accept(self)
+        node.trueblock.accept(self)
+        node.falseblock.accept(self)
+
+    def visit_ArgumentNode(self, node: mparser.ArgumentNode):
+        for i in node.arguments:
+            i.accept(self)
+        for i in node.commas:
+            pass
+        for val in node.kwargs.values():
+            val.accept(self)

mesonbuild/mintro.py

@@ -23,7 +23,7 @@ import json
 from . import build, coredata as cdata
 from . import environment
 from . import mesonlib
-from . import astinterpreter
+from .ast import AstInterpreter
 from . import mparser
 from . import mlog
 from . import compilers
@@ -158,7 +158,7 @@ class IntrospectionHelper:
         self.native_file = None
         self.cmd_line_options = {}
 
-class IntrospectionInterpreter(astinterpreter.AstInterpreter):
+class IntrospectionInterpreter(AstInterpreter):
     # Interpreter to detect the options without a build directory
     # Most of the code is stolen from interperter.Interpreter
     def __init__(self, source_root, subdir, backend, cross_file=None, subproject='', subproject_dir='subprojects', env=None):
@@ -183,20 +183,6 @@ class IntrospectionInterpreter(astinterpreter.AstInterpreter):
             'add_languages': self.func_add_languages
         })
 
-    def flatten_args(self, args):
-        # Resolve mparser.ArrayNode if needed
-        flattend_args = []
-        if isinstance(args, mparser.ArrayNode):
-            args = [x.value for x in args.args.arguments]
-        for i in args:
-            if isinstance(i, mparser.ArrayNode):
-                flattend_args += [x.value for x in i.args.arguments]
-            elif isinstance(i, str):
-                flattend_args += [i]
-            else:
-                pass
-        return flattend_args
-
     def func_project(self, node, args, kwargs):
         if len(args) < 1:
             raise InvalidArguments('Not enough arguments to project(). Needs at least the project name.')

mesonbuild/mparser.py

@@ -212,7 +212,15 @@ This will become a hard error in a future Meson release.""", self.getline(line_s
             if not matched:
                 raise ParseException('lexer', self.getline(line_start), lineno, col)
 
-class ElementaryNode:
+class BaseNode:
+    def accept(self, visitor):
+        fname = 'visit_{}'.format(type(self).__name__)
+        if hasattr(visitor, fname):
+            func = getattr(visitor, fname)
+            if hasattr(func, '__call__'):
+                func(self)
+
+class ElementaryNode(BaseNode):
     def __init__(self, token):
         self.lineno = token.lineno
         self.subdir = token.subdir
@@ -253,28 +261,28 @@ class ContinueNode(ElementaryNode):
 class BreakNode(ElementaryNode):
     pass
 
-class ArrayNode:
+class ArrayNode(BaseNode):
     def __init__(self, args):
         self.subdir = args.subdir
         self.lineno = args.lineno
         self.colno = args.colno
         self.args = args
 
-class DictNode:
+class DictNode(BaseNode):
     def __init__(self, args):
         self.subdir = args.subdir
         self.lineno = args.lineno
         self.colno = args.colno
         self.args = args
 
-class EmptyNode:
+class EmptyNode(BaseNode):
     def __init__(self, lineno, colno):
         self.subdir = ''
         self.lineno = lineno
         self.colno = colno
         self.value = None
 
-class OrNode:
+class OrNode(BaseNode):
     def __init__(self, left, right):
         self.subdir = left.subdir
         self.lineno = left.lineno
@@ -282,7 +290,7 @@ class OrNode:
         self.left = left
         self.right = right
 
-class AndNode:
+class AndNode(BaseNode):
     def __init__(self, left, right):
         self.subdir = left.subdir
         self.lineno = left.lineno
@@ -290,7 +298,7 @@ class AndNode:
         self.left = left
         self.right = right
 
-class ComparisonNode:
+class ComparisonNode(BaseNode):
     def __init__(self, ctype, left, right):
         self.lineno = left.lineno
         self.colno = left.colno
@@ -299,7 +307,7 @@ class ComparisonNode:
         self.right = right
         self.ctype = ctype
 
-class ArithmeticNode:
+class ArithmeticNode(BaseNode):
     def __init__(self, operation, left, right):
         self.subdir = left.subdir
         self.lineno = left.lineno
@@ -308,21 +316,21 @@ class ArithmeticNode:
         self.right = right
         self.operation = operation
 
-class NotNode:
+class NotNode(BaseNode):
     def __init__(self, location_node, value):
         self.subdir = location_node.subdir
         self.lineno = location_node.lineno
         self.colno = location_node.colno
         self.value = value
 
-class CodeBlockNode:
+class CodeBlockNode(BaseNode):
     def __init__(self, location_node):
         self.subdir = location_node.subdir
         self.lineno = location_node.lineno
         self.colno = location_node.colno
         self.lines = []
 
-class IndexNode:
+class IndexNode(BaseNode):
     def __init__(self, iobject, index):
         self.iobject = iobject
         self.index = index
@@ -330,7 +338,7 @@ class IndexNode:
         self.lineno = iobject.lineno
         self.colno = iobject.colno
 
-class MethodNode:
+class MethodNode(BaseNode):
     def __init__(self, subdir, lineno, colno, source_object, name, args):
         self.subdir = subdir
         self.lineno = lineno
@@ -340,7 +348,7 @@ class MethodNode:
         assert(isinstance(self.name, str))
         self.args = args
 
-class FunctionNode:
+class FunctionNode(BaseNode):
     def __init__(self, subdir, lineno, colno, func_name, args):
         self.subdir = subdir
         self.lineno = lineno
@@ -349,7 +357,7 @@ class FunctionNode:
         assert(isinstance(func_name, str))
         self.args = args
 
-class AssignmentNode:
+class AssignmentNode(BaseNode):
     def __init__(self, lineno, colno, var_name, value):
         self.lineno = lineno
         self.colno = colno
@@ -357,7 +365,7 @@ class AssignmentNode:
         assert(isinstance(var_name, str))
         self.value = value
 
-class PlusAssignmentNode:
+class PlusAssignmentNode(BaseNode):
     def __init__(self, lineno, colno, var_name, value):
         self.lineno = lineno
         self.colno = colno
@@ -365,7 +373,7 @@ class PlusAssignmentNode:
         assert(isinstance(var_name, str))
         self.value = value
 
-class ForeachClauseNode:
+class ForeachClauseNode(BaseNode):
     def __init__(self, lineno, colno, varnames, items, block):
         self.lineno = lineno
         self.colno = colno
@@ -373,28 +381,28 @@ class ForeachClauseNode:
         self.items = items
         self.block = block
 
-class IfClauseNode:
+class IfClauseNode(BaseNode):
     def __init__(self, lineno, colno):
         self.lineno = lineno
         self.colno = colno
         self.ifs = []
         self.elseblock = EmptyNode(lineno, colno)
 
-class UMinusNode:
+class UMinusNode(BaseNode):
     def __init__(self, current_location, value):
         self.subdir = current_location.subdir
         self.lineno = current_location.lineno
         self.colno = current_location.colno
         self.value = value
 
-class IfNode:
+class IfNode(BaseNode):
     def __init__(self, lineno, colno, condition, block):
         self.lineno = lineno
         self.colno = colno
         self.condition = condition
         self.block = block
 
-class TernaryNode:
+class TernaryNode(BaseNode):
     def __init__(self, lineno, colno, condition, trueblock, falseblock):
         self.lineno = lineno
         self.colno = colno
@@ -402,7 +410,7 @@ class TernaryNode:
         self.trueblock = trueblock
         self.falseblock = falseblock
 
-class ArgumentNode:
+class ArgumentNode(BaseNode):
     def __init__(self, token):
         self.lineno = token.lineno
         self.colno = token.colno

mesonbuild/rewriter.py

@@ -23,7 +23,7 @@
 # - move targets
 # - reindent?
 
-import mesonbuild.astinterpreter
+from .ast import (AstInterpreter, AstVisitor)
 from mesonbuild.mesonlib import MesonException
 from mesonbuild import mlog
 import sys, traceback
@@ -31,24 +31,20 @@ import sys, traceback
 def add_arguments(parser):
     parser.add_argument('--sourcedir', default='.',
                         help='Path to source directory.')
-    parser.add_argument('--target', default=None,
-                        help='Name of target to edit.')
-    parser.add_argument('--filename', default=None,
-                        help='Name of source file to add or remove to target.')
-    parser.add_argument('commands', nargs='+')
+    parser.add_argument('-p', '--print', action='store_true', default=False, dest='print',
+                        help='Print the parsed AST.')
 
 def run(options):
-    if options.target is None or options.filename is None:
-        sys.exit("Must specify both target and filename.")
     print('This tool is highly experimental, use with care.')
-    rewriter = mesonbuild.astinterpreter.RewriterInterpreter(options.sourcedir, '')
+    rewriter = AstInterpreter(options.sourcedir, '')
     try:
-        if options.commands[0] == 'add':
-            rewriter.add_source(options.target, options.filename)
-        elif options.commands[0] == 'remove':
-            rewriter.remove_source(options.target, options.filename)
-        else:
-            sys.exit('Unknown command: ' + options.commands[0])
+        rewriter.load_root_meson_file()
+        rewriter.sanity_check_ast()
+        rewriter.parse_project()
+        rewriter.run()
+
+        visitor = AstVisitor()
+        rewriter.ast.accept(visitor)
     except Exception as e:
         if isinstance(e, MesonException):
             mlog.exception(e)