msvc: Assume UTF8 source by default

Currently every project that uses UTF8 for its source files must add
'/utf-8' argument otherwise they don't work non-English locale MSVC.

Since meson.build itself is assumed to be UTF8 by default, seems better
to assume it for source files by default too.

For example:
- https://gitlab.freedesktop.org/gstreamer/gst-build/-/blob/master/meson.build#L62
- https://gitlab.gnome.org/GNOME/glib/-/blob/main/meson.build#L29

docs/markdown/snippets/msvc_utf8.md

@@ -0,0 +1,16 @@
+## MSVC compiler now assumes UTF-8 source code by default
+
+Every project that uses UTF-8 source files had to add manually `/utf-8` C/C++
+compiler argument for MSVC otherwise they wouldn't work on non-English locale.
+Meson now switched the default to UTF-8 to be more consistent with all other
+compilers.
+
+This can be overridden but using `/source-charset`:
+```meson
+if cc.get_id() == 'msvc'
+  add_project_arguments('/source-charset:.XYZ', language: ['c', 'cpp'])
+endif
+```
+
+See Microsoft documentation for details:
+https://docs.microsoft.com/en-us/cpp/build/reference/source-charset-set-source-character-set.

mesonbuild/compilers/mixins/visualstudio.py

@@ -100,7 +100,9 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta):
 
     # /showIncludes is needed for build dependency tracking in Ninja
     # See: https://ninja-build.org/manual.html#_deps
-    always_args = ['/nologo', '/showIncludes']
+    # Assume UTF-8 sources by default, but self.unix_args_to_native() removes it
+    # if `/source-charset` is set too.
+    always_args = ['/nologo', '/showIncludes', '/utf-8']
     warn_args = {
         '0': [],
         '1': ['/W2'],
@@ -214,7 +216,7 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta):
 
     @classmethod
     def unix_args_to_native(cls, args: T.List[str]) -> T.List[str]:
-        result = []
+        result: T.List[str] = []
         for i in args:
             # -mms-bitfields is specific to MinGW-GCC
             # -pthread is only valid for GCC
@@ -248,6 +250,13 @@ class VisualStudioLikeCompiler(Compiler, metaclass=abc.ABCMeta):
             # -pthread in link flags is only used on Linux
             elif i == '-pthread':
                 continue
+            # cl.exe does not allow specifying both, so remove /utf-8 that we
+            # added automatically in the case the user overrides it manually.
+            elif i.startswith('/source-charset:') or i.startswith('/execution-charset:'):
+                try:
+                    result.remove('/utf-8')
+                except ValueError:
+                    pass
             result.append(i)
         return result
 

test cases/windows/18 msvc charset/iso-8859-1.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char *argcv[])
+{
+  printf("This is ISO-8859-1 encoded é\n");
+  return 0;
+}

test cases/windows/18 msvc charset/meson.build

@@ -0,0 +1,15 @@
+project('charset', 'c')
+
+cc = meson.get_compiler('c')
+
+if cc.get_id() != 'msvc'
+  error('MESON_SKIP_TEST requires MSVC.')
+endif
+
+executable('utf8', 'utf8.c')
+
+if get_option('test-failure')
+  executable('iso-8859-1', 'iso-8859-1.c')
+else
+  executable('iso-8859-1', 'iso-8859-1.c', c_args: '/source-charset:.850')
+endif

test cases/windows/18 msvc charset/meson_options.txt

@@ -0,0 +1 @@
+option('test-failure', type: 'boolean', value: false)

test cases/windows/18 msvc charset/utf8.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char *argcv[])
+{
+  printf("This is UTF-8 encoded é\n");
+  return 0;
+}

unittests/windowstests.py

@@ -360,3 +360,16 @@ class WindowsTests(BasePlatformTests):
         self.init(os.path.join(self.unit_test_dir, '86 cpp modules'))
         self.build()
 
+    def test_non_utf8_fails(self):
+        # FIXME: VS backend does not use flags from compiler.get_always_args()
+        # and thus it's missing /utf-8 argument. Was that intentional? This needs
+        # to be revisited.
+        if self.backend is not Backend.ninja:
+            raise SkipTest(f'This test only pass with ninja backend (not {self.backend.name}).')
+        testdir = os.path.join(self.platform_test_dir, '18 msvc charset')
+        env = get_fake_env(testdir, self.builddir, self.prefix)
+        cc = detect_c_compiler(env, MachineChoice.HOST)
+        if cc.get_argument_syntax() != 'msvc':
+            raise SkipTest('Not using MSVC')
+        self.init(testdir, extra_args=['-Dtest-failure=true'])
+        self.assertRaises(subprocess.CalledProcessError, self.build)