raw printer

this printer preserves all whitespaces and comments in original meson.build file. It will be useful for rewrite and potential auto-formatter

docs/markdown/Syntax.md

@@ -109,7 +109,7 @@ Strings in Meson are declared with single quotes. To enter a literal
 single quote do it like this:
 
 ```meson
-single quote = 'contains a \' character'
+single_quote = 'contains a \' character'
 ```
 
 The full list of escape sequences is:

mesonbuild/ast/printer.py

@@ -18,6 +18,8 @@ from __future__ import annotations
 
 from .. import mparser
 from .visitor import AstVisitor
+
+from itertools import zip_longest
 import re
 import typing as T
 
@@ -248,6 +250,223 @@ class AstPrinter(AstVisitor):
         else:
             self.result = re.sub(r', $', '', self.result)
 
+class RawPrinter(AstVisitor):
+
+    def __init__(self):
+        self.result = ''
+
+    def visit_default_func(self, node: mparser.BaseNode):
+        self.result += node.value
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_unary_operator(self, node: mparser.UnaryOperatorNode):
+        node.operator.accept(self)
+        node.value.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_binary_operator(self, node: mparser.BinaryOperatorNode):
+        node.left.accept(self)
+        node.operator.accept(self)
+        node.right.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_BooleanNode(self, node: mparser.BooleanNode) -> None:
+        self.result += 'true' if node.value else 'false'
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_NumberNode(self, node: mparser.NumberNode) -> None:
+        self.result += node.raw_value
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_StringNode(self, node: mparser.StringNode) -> None:
+        self.result += f"'{node.raw_value}'"
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None:
+        self.result += f"'''{node.value}'''"
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
+        self.result += 'f'
+        self.visit_StringNode(node)
+
+    def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None:
+        self.result += 'f'
+        self.visit_MultilineStringNode(node)
+
+    def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
+        self.result += 'continue'
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_BreakNode(self, node: mparser.BreakNode) -> None:
+        self.result += 'break'
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_ArrayNode(self, node: mparser.ArrayNode) -> None:
+        node.lbracket.accept(self)
+        node.args.accept(self)
+        node.rbracket.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_DictNode(self, node: mparser.DictNode) -> None:
+        node.lcurl.accept(self)
+        node.args.accept(self)
+        node.rcurl.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_ParenthesizedNode(self, node: mparser.ParenthesizedNode) -> None:
+        node.lpar.accept(self)
+        node.inner.accept(self)
+        node.rpar.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_OrNode(self, node: mparser.OrNode) -> None:
+        self.visit_binary_operator(node)
+
+    def visit_AndNode(self, node: mparser.AndNode) -> None:
+        self.visit_binary_operator(node)
+
+    def visit_ComparisonNode(self, node: mparser.ComparisonNode) -> None:
+        self.visit_binary_operator(node)
+
+    def visit_ArithmeticNode(self, node: mparser.ArithmeticNode) -> None:
+        self.visit_binary_operator(node)
+
+    def visit_NotNode(self, node: mparser.NotNode) -> None:
+        self.visit_unary_operator(node)
+
+    def visit_CodeBlockNode(self, node: mparser.CodeBlockNode) -> None:
+        if node.pre_whitespaces:
+            node.pre_whitespaces.accept(self)
+        for i in node.lines:
+            i.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_IndexNode(self, node: mparser.IndexNode) -> None:
+        node.iobject.accept(self)
+        node.lbracket.accept(self)
+        node.index.accept(self)
+        node.rbracket.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_MethodNode(self, node: mparser.MethodNode) -> None:
+        node.source_object.accept(self)
+        node.dot.accept(self)
+        node.name.accept(self)
+        node.lpar.accept(self)
+        node.args.accept(self)
+        node.rpar.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_FunctionNode(self, node: mparser.FunctionNode) -> None:
+        node.func_name.accept(self)
+        node.lpar.accept(self)
+        node.args.accept(self)
+        node.rpar.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_AssignmentNode(self, node: mparser.AssignmentNode) -> None:
+        node.var_name.accept(self)
+        node.operator.accept(self)
+        node.value.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_PlusAssignmentNode(self, node: mparser.PlusAssignmentNode) -> None:
+        node.var_name.accept(self)
+        node.operator.accept(self)
+        node.value.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_ForeachClauseNode(self, node: mparser.ForeachClauseNode) -> None:
+        node.foreach_.accept(self)
+        for varname, comma in zip_longest(node.varnames, node.commas):
+            varname.accept(self)
+            if comma is not None:
+                comma.accept(self)
+        node.column.accept(self)
+        node.items.accept(self)
+        node.block.accept(self)
+        node.endforeach.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_IfClauseNode(self, node: mparser.IfClauseNode) -> None:
+        for i in node.ifs:
+            i.accept(self)
+        if not isinstance(node.elseblock, mparser.EmptyNode):
+            node.elseblock.accept(self)
+        node.endif.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_UMinusNode(self, node: mparser.UMinusNode) -> None:
+        self.visit_unary_operator(node)
+
+    def visit_IfNode(self, node: mparser.IfNode) -> None:
+        node.if_.accept(self)
+        node.condition.accept(self)
+        node.block.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_ElseNode(self, node: mparser.ElseNode) -> None:
+        node.else_.accept(self)
+        node.block.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_TernaryNode(self, node: mparser.TernaryNode) -> None:
+        node.condition.accept(self)
+        node.questionmark.accept(self)
+        node.trueblock.accept(self)
+        node.column.accept(self)
+        node.falseblock.accept(self)
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
+    def visit_ArgumentNode(self, node: mparser.ArgumentNode) -> None:
+        commas_iter = iter(node.commas)
+
+        for arg in node.arguments:
+            arg.accept(self)
+            try:
+                comma = next(commas_iter)
+                comma.accept(self)
+            except StopIteration:
+                pass
+
+        assert len(node.columns) == len(node.kwargs)
+        for (key, val), column in zip(node.kwargs.items(), node.columns):
+            key.accept(self)
+            column.accept(self)
+            val.accept(self)
+            try:
+                comma = next(commas_iter)
+                comma.accept(self)
+            except StopIteration:
+                pass
+
+        if node.whitespaces:
+            node.whitespaces.accept(self)
+
 class AstJSONPrinter(AstVisitor):
     def __init__(self) -> None:
         self.result: T.Dict[str, T.Any] = {}

run_format_tests.py

@@ -63,6 +63,7 @@ def check_format() -> None:
         'work area',
         '.eggs', '_cache',              # e.g. .mypy_cache
         'venv',                         # virtualenvs have DOS line endings
+        '118 rewrite',                  # we explicitly test for tab in meson.build file
     }
     for (root, _, filenames) in os.walk('.'):
         if any([x in root for x in skip_dirs]):

test cases/unit/118 rewrite/meson.build

@@ -0,0 +1,189 @@
+# This file should expose all possible meson syntaxes
+  #   and ensure the AstInterpreter and RawPrinter are able
+  
+  # to parse and write a file identical to the original.
+
+  project  (       # project comment 1
+    # project comment 2
+    'rewrite'  ,  # argument comment
+    # project comment 3
+        'cpp',
+    'c',
+    default_options: [
+        'unity=on',
+        'unity_size=50',  # number of cpp / unity. default is 4...
+        'warning_level=2',  # eqv to /W3
+        'werror=true',  # treat warnings as errors
+        'b_ndebug=if-release',  # disable assert in Release
+        'cpp_eh=a',  # /EHa exception handling
+        'cpp_std=c++17',
+        'cpp_winlibs=' + ','.join([ # array comment
+        # in array
+          # comment
+            'kernel32.lib',
+            'user32.lib',
+            'gdi32.lib',
+            'winspool.lib',
+            'comdlg32.lib',
+            'advapi32.lib',
+            'shell32.lib'
+            # before comma comment
+            ,
+            # after comma comment
+            'ole32.lib',
+            'oleaut32.lib',
+            'uuid.lib',
+            'odbc32.lib',
+            'odbccp32.lib',
+            'Delayimp.lib',  # For delay loaded dll
+            'OLDNAMES.lib',
+            'dbghelp.lib',
+            'psapi.lib',
+        ]),
+    ],
+    meson_version: '>=1.2',
+    version: '1.0.0',
+ )  # project comment 4
+
+cppcoro_dep = dependency('andreasbuhr-cppcoro-cppcoro')
+cppcoro = declare_dependency(
+    dependencies: [cppcoro_dep.partial_dependency(
+        includes: true,
+        link_args: true,
+        links: true,
+        sources: true,
+    )],
+    # '/await:strict' allows to use <coroutine> rather than <experimental/coroutine> with C++17.
+    # We can remove '/await:strict' once we update to C++20.
+    compile_args: ['/await:strict'],
+    # includes:true doesn't work for now in partial_dependency()
+    # This line could be removed once https://github.com/mesonbuild/meson/pull/10122 is released.
+    include_directories: cppcoro_dep.get_variable('includedir1'),
+)
+
+
+if get_option('unicode')  #if comment
+#if comment 2
+  mfc=cpp_compiler.find_library(get_option('debug')?'mfc140ud':'mfc140u')
+       # if comment 3
+else#elsecommentnowhitespaces
+       # else  comment 1
+  mfc = cpp_compiler.find_library( get_option( 'debug' ) ? 'mfc140d' : 'mfc140')
+# else comment 2
+endif  #endif comment
+
+
+assert(1 in [1, 2], '''1 should be in [1, 2]''')
+assert(3 not in [1, 2], '''3 shouldn't be in [1, 2]''')
+assert(not (3 in [1, 2]), '''3 shouldn't be in [1, 2]''')
+
+assert('b' in ['a', 'b'], ''''b' should be in ['a', 'b']''')
+assert('c' not in ['a', 'b'], ''''c' shouldn't be in ['a', 'b']''')
+
+assert(exe1 in [exe1, exe2], ''''exe1 should be in [exe1, exe2]''')
+assert(exe3 not in [exe1, exe2], ''''exe3 shouldn't be in [exe1, exe2]''')
+
+assert('a' in {'a': 'b'}, '''1 should be in {'a': 'b'}''')
+assert('b'not in{'a':'b'}, '''1 should be in {'a': 'b'}''')
+
+assert('a'in'abc')
+assert('b'  not   in    'def')
+
+
+w = 'world'
+d = {'a': 1, 'b': 0b10101010, 'c': 'pi', 'd': '''a
+b
+c''', 'e': f'hello @w@', 'f': f'''triple
+  formatted
+    string # this is not a comment
+      hello @w@
+''', 'g': [1, 2, 3], 
+
+    'h' # comment a
+    : # comment b
+0xDEADBEEF # comment c
+, # comment d
+'hh': 0xfeedc0de, # lowercase hexa
+'hhh': 0XaBcD0123,  # mixed case hexa
+'oo': 0O123456,  # upper O octa
+'bb': 0B1111,  # upper B binary
+'i': {'aa': 11, # this is a comment
+  'bb': 22}, # a comment inside a dict
+'o': 0o754,
+'m': -12,  # minus number
+'eq': 1 + 3 - 3 % 4 + -( 7 * 8 ),
+}  # end of dict comment
+
+hw = d['e']
+one = d['g'][0]
+  w  +=  '!'
+
+
+components = {
+  'foo': ['foo.c'],
+  'bar': ['bar.c'],
+	'baz': ['baz.c'], # this line is indented with a tab!
+}
+
+# compute a configuration based on system dependencies, custom logic
+conf = configuration_data()
+conf.set('USE_FOO', 1)
+
+# Determine the sources to compile
+sources_to_compile = []
+foreach name, sources : components
+  if conf.get('USE_@0@'.format(name.to_upper()), 0) == 1
+    sources_to_compile += sources
+  endif
+endforeach
+
+
+items = ['a', 'continue', 'b', 'break', 'c']
+result = []
+foreach i : items
+  if i == 'continue'
+    continue
+  elif i == 'break'
+    break
+  endif
+  result += i
+endforeach
+# result is ['a', 'b']
+
+
+
+if a and b
+  # do something
+endif
+if c or d
+  # do something
+endif
+if not e
+  # do something
+endif
+if not (f or g)
+  # do something
+endif
+
+single_quote = 'contains a \' character'
+string_escapes = '\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}'
+no_string_escapes = '''\\\'\a\b\f\n\r\t\v\046\x26\u2D4d\U00002d4d\N{GREEK CAPITAL LETTER DELTA}'''
+
+# FIXME: is it supposed to work? (cont_eol inside string)
+# cont_string = 'blablabla\
+# blablabla'
+
+# FIXME: is it supposed to work? (cont_eol with whitespace and comments after)
+# if a \  # comment in cont 1
+#     and b \  # comment in cont 2
+#     or c  # comment in cont 3
+#   message('ok')
+# endif
+
+if a \
+    or b
+  debug('help!')
+endif
+
+
+# End of file comment with no linebreak

unittests/rewritetests.py

@@ -13,11 +13,15 @@
 # limitations under the License.
 
 import subprocess
+from itertools import zip_longest
 import json
 import os
+from pathlib import Path
 import shutil
 import unittest
 
+from mesonbuild.ast import IntrospectionInterpreter, AstIDGenerator
+from mesonbuild.ast.printer import RawPrinter
 from mesonbuild.mesonlib import windows_proof_rmtree
 from .baseplatformtests import BasePlatformTests
 
@@ -396,3 +400,21 @@ class RewriterTests(BasePlatformTests):
         # Check the written file
         out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
         self.assertDictEqual(out, expected)
+
+    def test_raw_printer_is_idempotent(self):
+        test_path = Path(self.unit_test_dir, '118 rewrite')
+        meson_build_file = test_path / 'meson.build'
+        # original_contents = meson_build_file.read_bytes()
+        original_contents = meson_build_file.read_text(encoding='utf-8')
+
+        interpreter = IntrospectionInterpreter(test_path, '', 'ninja', visitors = [AstIDGenerator()])
+        interpreter.analyze()
+
+        printer = RawPrinter()
+        interpreter.ast.accept(printer)
+        # new_contents = printer.result.encode('utf-8')
+        new_contents = printer.result
+
+        # Do it line per line because it is easier to debug like that
+        for orig_line, new_line in zip_longest(original_contents.splitlines(), new_contents.splitlines()):
+            self.assertEqual(orig_line, new_line)