Detect utf8 bom from meson build files

Some text editors on Windows may use utf8bom encoding by default.
Prevent crash and properly report misencoded files.

Fixes #12766.

mesonbuild/interpreterbase/interpreterbase.py

@@ -107,10 +107,11 @@ class InterpreterBase:
             self.handle_meson_version_from_ast()
         except mparser.ParseException as me:
             me.file = mesonfile
-            # try to detect parser errors from new syntax added by future
-            # meson versions, and just tell the user to update meson
-            self.ast = me.ast
-            self.handle_meson_version_from_ast()
+            if me.ast:
+                # try to detect parser errors from new syntax added by future
+                # meson versions, and just tell the user to update meson
+                self.ast = me.ast
+                self.handle_meson_version_from_ast()
             raise me
 
     def parse_project(self) -> None:

mesonbuild/mparser.py

@@ -96,6 +96,10 @@ class Token(T.Generic[TV_TokenTypes]):
 
 class Lexer:
     def __init__(self, code: str):
+        if code.startswith(codecs.BOM_UTF8.decode('utf-8')):
+            line, *_ = code.split('\n', maxsplit=1)
+            raise ParseException('Builder file must be encoded in UTF-8 (with no BOM)', line, lineno=0, colno=0)
+
         self.code = code
         self.keywords = {'true', 'false', 'if', 'else', 'elif',
                          'endif', 'and', 'or', 'not', 'foreach', 'endforeach',

test cases/failing/130 utf8 with bom/meson.build

@@ -0,0 +1,3 @@
+project('utf8 with bom')
+
+subdir('subdir')

test cases/failing/130 utf8 with bom/test.json

@@ -0,0 +1,8 @@
+{
+    "stdout": [
+      {
+        "line": "test cases/failing/130 utf8 with bom/meson.build:0:0: ERROR: Builder file must be encoded in UTF-8 (with no BOM)"
+      }
+    ]
+  }
+  

test cases/failing/131 utf8 with bom subdir/meson.build

@@ -0,0 +1,3 @@
+project('utf8 with bom subdir')
+
+subdir('subdir')

test cases/failing/131 utf8 with bom subdir/subdir/meson.build

@@ -0,0 +1 @@
+a = 'Hello, World!'

test cases/failing/131 utf8 with bom subdir/test.json

@@ -0,0 +1,8 @@
+{
+    "stdout": [
+      {
+        "line": "test cases/failing/131 utf8 with bom subdir/subdir/meson.build:0:0: ERROR: Builder file must be encoded in UTF-8 (with no BOM)"
+      }
+    ]
+  }
+  

test cases/failing/132 utf8 with bom options/meson.build

@@ -0,0 +1 @@
+project('utf8 with bom options')

test cases/failing/132 utf8 with bom options/meson.options

@@ -0,0 +1 @@
+option('someoption', type : 'string', value : 'optval', description : 'An option')

test cases/failing/132 utf8 with bom options/test.json

@@ -0,0 +1,8 @@
+{
+    "stdout": [
+      {
+        "line": "test cases/failing/132 utf8 with bom options/meson.options:0:0: ERROR: Builder file must be encoded in UTF-8 (with no BOM)"
+      }
+    ]
+  }
+