Store subdir information for each node so we can remove files set in other subdirectories.

mesonbuild/astinterpreter.py

@@ -46,6 +46,7 @@ REMOVE_SOURCE = 1
 class AstInterpreter(interpreterbase.InterpreterBase):
     def __init__(self, source_root, subdir):
         super().__init__(source_root, subdir)
+        self.asts = {}
         self.funcs.update({'project' : self.func_do_nothing,
                            'test' : self.func_do_nothing,
                            'benchmark' : self.func_do_nothing,
@@ -133,7 +134,8 @@ class AstInterpreter(interpreterbase.InterpreterBase):
             code = f.read()
         assert(isinstance(code, str))
         try:
-            codeblock = mparser.Parser(code).parse()
+            codeblock = mparser.Parser(code, self.subdir).parse()
+            self.asts[subdir] = codeblock
         except mesonlib.MesonException as me:
             me.file = buildfilename
             raise me
@@ -162,6 +164,7 @@ class AstInterpreter(interpreterbase.InterpreterBase):
 
     def transform(self):
         self.load_root_meson_file()
+        self.asts[''] = self.ast
         self.sanity_check_ast()
         self.parse_project()
         self.run()
@@ -202,7 +205,7 @@ class AstInterpreter(interpreterbase.InterpreterBase):
             commaspan = args.commas[i].bytespan
         if commaspan[0] < namespan[0]:
             commaspan, namespan = namespan, commaspan
-        buildfilename = os.path.join(self.source_root, self.subdir, environment.build_filename)
+        buildfilename = os.path.join(self.source_root, args.subdir, environment.build_filename)
         raw_data = open(buildfilename, 'r').read()
         intermediary = raw_data[0:commaspan[0]] + raw_data[commaspan[1]:]
         updated = intermediary[0:namespan[0]] + intermediary[namespan[1]:]
@@ -210,7 +213,7 @@ class AstInterpreter(interpreterbase.InterpreterBase):
         sys.exit(0)
 
     def hacky_find_and_remove(self, node_to_remove):
-        for a in self.ast.lines:
+        for a in self.asts[node_to_remove.subdir].lines:
             if a.lineno == node_to_remove.lineno:
                 if isinstance(a, mparser.AssignmentNode):
                     v = a.value

mesonbuild/interpreter.py

@@ -1991,7 +1991,7 @@ requirements use the version keyword argument instead.''')
             code = f.read()
         assert(isinstance(code, str))
         try:
-            codeblock = mparser.Parser(code).parse()
+            codeblock = mparser.Parser(code, self.subdir).parse()
         except mesonlib.MesonException as me:
             me.file = buildfilename
             raise me

mesonbuild/interpreterbase.py

@@ -97,7 +97,7 @@ class InterpreterBase:
             raise InvalidCode('Builder file is empty.')
         assert(isinstance(code, str))
         try:
-            self.ast = mparser.Parser(code).parse()
+            self.ast = mparser.Parser(code, self.subdir).parse()
         except mesonlib.MesonException as me:
             me.file = environment.build_filename
             raise me

mesonbuild/mparser.py

@@ -22,8 +22,9 @@ class ParseException(MesonException):
         self.colno = colno
 
 class Token:
-    def __init__(self, tid, lineno, colno, bytespan, value):
+    def __init__(self, tid, subdir, lineno, colno, bytespan, value):
         self.tid = tid
+        self.subdir = subdir
         self.lineno = lineno
         self.colno = colno
         self.bytespan = bytespan
@@ -72,7 +73,7 @@ class Lexer:
             ('questionmark', re.compile(r'\?')),
         ]
 
-    def lex(self, code):
+    def lex(self, code, subdir):
         lineno = 1
         line_start = 0
         loc = 0;
@@ -127,7 +128,7 @@ class Lexer:
                             tid = match_text
                         else:
                             value = match_text
-                    yield Token(tid, curline, col, bytespan, value)
+                    yield Token(tid, subdir, curline, col, bytespan, value)
                     break
             if not matched:
                 raise ParseException('lexer', lineno, col)
@@ -135,6 +136,7 @@ class Lexer:
 class ElementaryNode:
     def __init__(self, token):
         self.lineno = token.lineno
+        self.subdir = token.subdir
         self.colno = token.colno
         self.value = token.value
         self.bytespan = token.bytespan
@@ -168,20 +170,23 @@ class StringNode(ElementaryNode):
 
 class ArrayNode:
     def __init__(self, args):
+        self.subdir = args.subdir
         self.lineno = args.lineno
         self.colno = args.colno
         self.args = args
 
 class EmptyNode:
     def __init__(self):
+        self.subdir =''
         self.lineno = 0
         self.colno = 0
         self.value = None
 
 class OrNode:
-    def __init__(self, lineno, colno, left, right):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self, left, right):
+        self.subdir = left.subdir
+        self.lineno = left.lineno
+        self.colno = left.colno
         self.left = left
         self.right = right
 
@@ -193,42 +198,48 @@ class AndNode:
         self.right = right
 
 class ComparisonNode:
-    def __init__(self, lineno, colno, ctype, left, right):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self, ctype, left, right):
+        self.lineno = left.lineno
+        self.colno = left.colno
+        self.subdir = left.subdir
         self.left = left
         self.right = right
         self.ctype = ctype
 
 class ArithmeticNode:
-    def __init__(self, lineno, colno, operation, left, right):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self,operation, left, right):
+        self.subdir = left.subdir
+        self.lineno = left.lineno
+        self.colno = left.colno
         self.left = left
         self.right = right
         self.operation = operation
 
 class NotNode:
-    def __init__(self, lineno, colno, value):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self, location_node, value):
+        self.subdir = location_node.subdir
+        self.lineno = location_node.lineno
+        self.colno = location_node.colno
         self.value = value
 
 class CodeBlockNode:
-    def __init__(self, lineno, colno):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self, location_node):
+        self.subdir = location_node.subdir
+        self.lineno = location_node.lineno
+        self.colno = location_node.colno
         self.lines = []
 
 class IndexNode:
     def __init__(self, iobject, index):
         self.iobject = iobject
         self.index = index
+        self.subdir = iobject.subdir
         self.lineno = iobject.lineno
         self.colno = iobject.colno
 
 class MethodNode:
-    def __init__(self, lineno, colno, source_object, name, args):
+    def __init__(self, subdir, lineno, colno, source_object, name, args):
+        self.subdir = subdir
         self.lineno = lineno
         self.colno = colno
         self.source_object = source_object
@@ -237,7 +248,8 @@ class MethodNode:
         self.args = args
 
 class FunctionNode:
-    def __init__(self, lineno, colno, func_name, args):
+    def __init__(self, subdir, lineno, colno, func_name, args):
+        self.subdir = subdir
         self.lineno = lineno
         self.colno = colno
         self.func_name = func_name
@@ -276,9 +288,10 @@ class IfClauseNode():
         self.elseblock = EmptyNode()
 
 class UMinusNode():
-    def __init__(self, lineno, colno, value):
-        self.lineno = lineno
-        self.colno = colno
+    def __init__(self, current_location, value):
+        self.subdir = current_location.subdir
+        self.lineno = current_location.lineno
+        self.colno = current_location.colno
         self.value = value
 
 class IfNode():
@@ -300,6 +313,7 @@ class ArgumentNode():
     def __init__(self, token):
         self.lineno = token.lineno
         self.colno = token.colno
+        self.subdir = token.subdir
         self.arguments = []
         self.commas = []
         self.kwargs = {}
@@ -356,8 +370,8 @@ comparison_map = {'equal': '==',
 # 9 plain token
 
 class Parser:
-    def __init__(self, code):
-        self.stream = Lexer().lex(code)
+    def __init__(self, code, subdir):
+        self.stream = Lexer().lex(code, subdir)
         self.getsym()
         self.in_ternary = False
 
@@ -365,7 +379,7 @@ class Parser:
         try:
             self.current = next(self.stream)
         except StopIteration:
-            self.current = Token('eof', 0, 0, (0, 0), None)
+            self.current = Token('eof', '', 0, 0, (0, 0), None)
 
     def accept(self, s):
         if self.current.tid == s:
@@ -414,7 +428,7 @@ class Parser:
     def e2(self):
         left = self.e3()
         while self.accept('or'):
-            left = OrNode(left.lineno, left.colno, left, self.e3())
+            left = OrNode(left, self.e3())
         return left
 
     def e3(self):
@@ -427,7 +441,7 @@ class Parser:
         left = self.e5()
         for nodename, operator_type in comparison_map.items():
             if self.accept(nodename):
-                return ComparisonNode(left.lineno, left.colno, operator_type, left, self.e5())
+                return ComparisonNode(operator_type, left, self.e5())
         return left
 
     def e5(self):
@@ -436,38 +450,38 @@ class Parser:
     def e5add(self):
         left = self.e5sub()
         if self.accept('plus'):
-            return ArithmeticNode(left.lineno, left.colno, 'add', left, self.e5add())
+            return ArithmeticNode('add', left, self.e5add())
         return left
 
     def e5sub(self):
         left = self.e5mod()
         if self.accept('dash'):
-            return ArithmeticNode(left.lineno, left.colno, 'sub', left, self.e5sub())
+            return ArithmeticNode('sub', left, self.e5sub())
         return left
 
     def e5mod(self):
         left = self.e5mul()
         if self.accept('percent'):
-            return ArithmeticNode(left.lineno, left.colno, 'mod', left, self.e5mod())
+            return ArithmeticNode('mod', left, self.e5mod())
         return left
 
     def e5mul(self):
         left = self.e5div()
         if self.accept('star'):
-            return ArithmeticNode(left.lineno, left.colno, 'mul', left, self.e5mul())
+            return ArithmeticNode('mul', left, self.e5mul())
         return left
 
     def e5div(self):
         left = self.e6()
         if self.accept('fslash'):
-            return ArithmeticNode(left.lineno, left.colno, 'div', left, self.e5div())
+            return ArithmeticNode('div', left, self.e5div())
         return left
 
     def e6(self):
         if self.accept('not'):
-            return NotNode(self.current.lineno, self.current.colno, self.e7())
+            return NotNode(self.current, self.e7())
         if self.accept('dash'):
-            return UMinusNode(self.current.lineno, self.current.colno, self.e7())
+            return UMinusNode(self.current, self.e7())
         return self.e7()
 
     def e7(self):
@@ -478,7 +492,7 @@ class Parser:
             if not isinstance(left, IdNode):
                 raise ParseException('Function call must be applied to plain id',
                                      left.lineno, left.colno)
-            left = FunctionNode(left.lineno, left.colno, left.value, args)
+            left = FunctionNode(left.subdir, left.lineno, left.colno, left.value, args)
         go_again = True
         while go_again:
             go_again = False
@@ -548,7 +562,7 @@ class Parser:
         self.expect('lparen')
         args = self.args()
         self.expect('rparen')
-        method = MethodNode(methodname.lineno, methodname.colno, source_object, methodname.value, args)
+        method = MethodNode(methodname.subdir, methodname.lineno, methodname.colno, source_object, methodname.value, args)
         if self.accept('dot'):
             return self.method_call(method)
         return method
@@ -602,7 +616,7 @@ class Parser:
         return self.statement()
 
     def codeblock(self):
-        block = CodeBlockNode(self.current.lineno, self.current.colno)
+        block = CodeBlockNode(self.current)
         cond = True
         while cond:
             curline = self.line()

mesonbuild/optinterpreter.py

@@ -79,7 +79,7 @@ class OptionInterpreter:
     def process(self, option_file):
         try:
             with open(option_file, 'r', encoding='utf8') as f:
-                ast = mparser.Parser(f.read()).parse()
+                ast = mparser.Parser(f.read(), '').parse()
         except mesonlib.MesonException as me:
             me.file = option_file
             raise me

mesonrewriter.py

@@ -44,12 +44,12 @@ if __name__ == '__main__':
     if options.target is None or options.filename is None:
         sys.exit("Must specify both target and filename.")
     print('This tool is highly experimental, use with care.')
-    ast = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
+    rewriter = mesonbuild.astinterpreter.AstInterpreter(options.sourcedir, '')
     try:
         if options.commands[0] == 'add':
-            ast.add_source(options.target, options.filename)
+            rewriter.add_source(options.target, options.filename)
         elif options.commands[0] == 'remove':
-            ast.remove_source(options.target, options.filename)
+            rewriter.remove_source(options.target, options.filename)
         else:
             sys.exit('Unknown command: ' + options.commands[0])
     except Exception as e:

run_unittests.py

@@ -218,17 +218,22 @@ class RewriterTests(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.tmpdir)
 
+    def read_contents(self, fname):
+        with open(os.path.join(self.workdir, fname)) as f:
+            return f.read()
+
     def check_effectively_same(self, mainfile, truth):
-        with open(os.path.join(self.workdir, mainfile)) as f:
-            mf = f.read()
-        with open(os.path.join(self.workdir, truth)) as f:
-            t = f.read()
+        mf = self.read_contents(mainfile)
+        t = self.read_contents(truth)
         # Rewriting is not guaranteed to do a perfect job of
         # maintaining whitespace.
         self.assertEqual(mf.replace(' ', ''), t.replace(' ', ''))
 
+    def prime(self, dirname):
+        shutil.copytree(os.path.join(self.test_dir, dirname), self.workdir)
+
     def test_basic(self):
-        shutil.copytree(os.path.join(self.test_dir, '1 basic/'), self.workdir)
+        self.prime('1 basic')
         subprocess.check_output(self.rewrite_command + ['remove',
                                                         '--target=trivialprog',
                                                         '--filename=notthere.c',
@@ -245,5 +250,17 @@ class RewriterTests(unittest.TestCase):
                                                         '--sourcedir', self.workdir])
         self.check_effectively_same('meson.build', 'removed.txt')
 
+    def test_subdir(self):
+        self.prime('2 subdirs')
+        top = self.read_contents('meson.build')
+        s2 = self.read_contents('sub2/meson.build')
+        subprocess.check_output(self.rewrite_command + ['remove',
+                                                        '--target=something',
+                                                        '--filename=second.c',
+                                                        '--sourcedir', self.workdir])
+        self.check_effectively_same('sub1/meson.build', 'sub1/after.txt')
+        self.assertEqual(top, self.read_contents('meson.build'))
+        self.assertEqual(s2, self.read_contents('sub2/meson.build'))
+
 if __name__ == '__main__':
     unittest.main()

test cases/rewrite/2 subdirs/meson.build

@@ -0,0 +1,5 @@
+project('subdir rewrite', 'c')
+
+subdir('sub1')
+subdir('sub2')
+

test cases/rewrite/2 subdirs/sub1/after.txt

@@ -0,0 +1 @@
+srcs = ['first.c']

test cases/rewrite/2 subdirs/sub1/meson.build

@@ -0,0 +1 @@
+srcs = ['first.c', 'second.c']

test cases/rewrite/2 subdirs/sub2/meson.build

@@ -0,0 +1,2 @@
+executable('something', srcs)
+