Add file encoding to configure_file

Input files can be in any file encoding, not just utf-8 or isolatin1. Meson
should not make assumptions here and allow for the user to specify the
encoding to use.

mesonbuild/interpreter.py

@@ -1724,7 +1724,7 @@ permitted_kwargs = {'add_global_arguments': {'language'},
                     'add_test_setup': {'exe_wrapper', 'gdb', 'timeout_multiplier', 'env'},
                     'benchmark': {'args', 'env', 'should_fail', 'timeout', 'workdir', 'suite'},
                     'build_target': known_build_target_kwargs,
-                    'configure_file': {'input', 'output', 'configuration', 'command', 'copy', 'install_dir', 'install_mode', 'capture', 'install', 'format', 'output_format'},
+                    'configure_file': {'input', 'output', 'configuration', 'command', 'copy', 'install_dir', 'install_mode', 'capture', 'install', 'format', 'output_format', 'encoding'},
                     'custom_target': {'input', 'output', 'command', 'install', 'install_dir', 'install_mode', 'build_always', 'capture', 'depends', 'depend_files', 'depfile', 'build_by_default'},
                     'dependency': {'default_options', 'fallback', 'language', 'main', 'method', 'modules', 'optional_modules', 'native', 'required', 'static', 'version', 'private_headers'},
                     'declare_dependency': {'include_directories', 'link_with', 'sources', 'dependencies', 'compile_args', 'link_args', 'link_whole', 'version'},
@@ -3332,8 +3332,10 @@ root and issuing %s.
             mlog.log('Configuring', mlog.bold(output), 'using configuration')
             if inputfile is not None:
                 os.makedirs(os.path.join(self.environment.build_dir, self.subdir), exist_ok=True)
+                file_encoding = kwargs.setdefault('encoding', 'utf-8')
                 missing_variables = mesonlib.do_conf_file(ifile_abs, ofile_abs,
-                                                          conf.held_object, fmt)
+                                                          conf.held_object, fmt,
+                                                          file_encoding)
                 if missing_variables:
                     var_list = ", ".join(map(repr, sorted(missing_variables)))
                     mlog.warning(
@@ -3360,7 +3362,8 @@ root and issuing %s.
                                            (res.stdout, res.stderr))
             if 'capture' in kwargs and kwargs['capture']:
                 dst_tmp = ofile_abs + '~'
-                with open(dst_tmp, 'w', encoding='utf-8') as f:
+                file_encoding = kwargs.setdefault('encoding', 'utf-8')
+                with open(dst_tmp, 'w', encoding=file_encoding) as f:
                     f.writelines(res.stdout)
                 if ifile_abs:
                     shutil.copymode(ifile_abs, dst_tmp)

mesonbuild/mesonlib.py

@@ -623,9 +623,9 @@ def do_mesondefine(line, confdata):
         raise MesonException('#mesondefine argument "%s" is of unknown type.' % varname)
 
 
-def do_conf_file(src, dst, confdata, format):
+def do_conf_file(src, dst, confdata, format, encoding='utf-8'):
     try:
-        with open(src, encoding='utf-8') as f:
+        with open(src, encoding=encoding) as f:
             data = f.readlines()
     except Exception as e:
         raise MesonException('Could not read input file %s: %s' % (src, str(e)))
@@ -652,8 +652,11 @@ def do_conf_file(src, dst, confdata, format):
             missing_variables.update(missing)
         result.append(line)
     dst_tmp = dst + '~'
-    with open(dst_tmp, 'w', encoding='utf-8') as f:
-        f.writelines(result)
+    try:
+        with open(dst_tmp, 'w', encoding=encoding) as f:
+            f.writelines(result)
+    except Exception as e:
+        raise MesonException('Could not write output file %s: %s' % (dst, str(e)))
     shutil.copymode(src, dst_tmp)
     replace_if_different(dst, dst_tmp)
     return missing_variables

test cases/common/16 configure file/config8.h.in

@@ -0,0 +1,3 @@
+#define MESSAGE "@var@"
+
+#define "non isolatin1 char Ä fails decode with utf-8"

test cases/common/16 configure file/meson.build

@@ -174,3 +174,13 @@ ret = run_command(check_file, inf, outf)
 if ret.returncode() != 0
   error('Error running command: @0@\n@1@'.format(ret.stdout(), ret.stderr()))
 endif
+
+# Test non isolatin1 encoded input file which fails to decode with utf-8
+conf8 = configuration_data()
+conf8.set('var', 'foo')
+configure_file(
+  input : 'config8.h.in',
+  output : '@BASENAME@',
+  encoding : 'koi8-r',
+  configuration : conf8
+)