i18n: Add support for joining XML localization via itstool

docs/markdown/i18n-module.md

@@ -56,3 +56,15 @@ for normal keywords. In addition it accepts these keywords:
 * `args`: (*Added 0.51.0*) list of extra arguments to pass to `msgfmt`
 
 *Added 0.37.0*
+
+### i18n.itstool_join()
+
+This joins translations into a XML file using `itstool`. See
+[[@custom_tgt]]
+for normal keywords. In addition it accepts these keywords:
+
+* `its_files`: filenames of ITS files that should be used explicitly
+  (XML translation rules are autodetected otherwise).
+* `mo_targets` *required*: mo file generation targets as returned by `i18n.gettext()`.
+
+*Added 0.61.0*

docs/markdown/snippets/i18n-itstool_join-added.md

@@ -0,0 +1,5 @@
+## Added support for XML translations using itstool
+
+XML files can now be translated easier by using `itstool` via
+`i18n.itstool_join()`. This ensures the XML is translated correctly
+based on the defined ITS rules for the specific XML layout.

mesonbuild/modules/i18n.py

@@ -59,6 +59,20 @@ if T.TYPE_CHECKING:
         languages: T.List[str]
         preset: T.Optional[str]
 
+    class ItsJoinFile(TypedDict):
+
+        input: T.List[T.Union[
+            str, build.BuildTarget, build.CustomTarget, build.CustomTargetIndex,
+            build.ExtractedObjects, build.GeneratedList, ExternalProgram,
+            mesonlib.File]]
+        output: T.List[str]
+        build_by_default: bool
+        install: bool
+        install_dir: T.List[T.Union[str, bool]]
+        install_tag: T.List[str]
+        its_files: T.List[str]
+        mo_targets: T.List[T.Union[build.BuildTarget, build.CustomTarget, build.CustomTargetIndex]]
+
 
 _ARGS: KwargInfo[T.List[str]] = KwargInfo(
     'args',
@@ -115,12 +129,17 @@ class I18nModule(ExtensionModule):
         self.methods.update({
             'merge_file': self.merge_file,
             'gettext': self.gettext,
+            'itstool_join': self.itstool_join,
         })
 
     @staticmethod
     def nogettext_warning() -> None:
         mlog.warning('Gettext not found, all translation targets will be ignored.', once=True)
 
+    @staticmethod
+    def noitstool_error() -> T.NoReturn:
+        raise mesonlib.MesonException('Did not find itstool. Please install it to continue.')
+
     @staticmethod
     def _get_data_dirs(state: 'ModuleState', dirs: T.Iterable[str]) -> T.List[str]:
         """Returns source directories of relative paths"""
@@ -269,5 +288,65 @@ class I18nModule(ExtensionModule):
 
         return ModuleReturnValue([gmotargets, pottarget, updatepotarget], targets)
 
+    @FeatureNew('i18n.itstool_join', '0.61.0')
+    @noPosargs
+    @typed_kwargs(
+        'i18n.itstool_join',
+        CT_BUILD_BY_DEFAULT,
+        CT_INPUT_KW,
+        CT_INSTALL_DIR_KW,
+        CT_INSTALL_TAG_KW,
+        CT_OUTPUT_KW,
+        INSTALL_KW,
+        _ARGS.evolve(),
+        KwargInfo('its_files', ContainerTypeInfo(list, str)),
+        KwargInfo('mo_targets', ContainerTypeInfo(list, build.CustomTarget), required=True),
+    )
+    def itstool_join(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'ItsJoinFile') -> ModuleReturnValue:
+        if not shutil.which('itstool'):
+            self.noitstool_error()
+        mo_targets = kwargs['mo_targets']
+        its_files = kwargs.get('its_files', [])
+
+        mo_fnames = []
+        for target in mo_targets:
+            mo_fnames.append(path.join(target.get_subdir(), target.get_outputs()[0]))
+
+        command: T.List[T.Union[str, build.BuildTarget, build.CustomTarget,
+                                build.CustomTargetIndex, 'ExternalProgram', mesonlib.File]] = []
+        command.extend(state.environment.get_build_command())
+        command.extend([
+            '--internal', 'itstool', 'join',
+            '-i', '@INPUT@',
+            '-o', '@OUTPUT@'
+        ])
+        if its_files:
+            for fname in its_files:
+                if not path.isabs(fname):
+                    fname = path.join(state.environment.source_dir, state.subdir, fname)
+                command.extend(['--its', fname])
+        command.extend(mo_fnames)
+
+        build_by_default = kwargs['build_by_default']
+        if build_by_default is None:
+            build_by_default = kwargs['install']
+
+        real_kwargs = {
+            'build_by_default': build_by_default,
+            'command': command,
+            'depends': mo_targets,
+            'install': kwargs['install'],
+            'install_dir': kwargs['install_dir'],
+            'output': kwargs['output'],
+            'input': kwargs['input'],
+            'install_tag': kwargs['install_tag'],
+        }
+
+        ct = build.CustomTarget('', state.subdir, state.subproject,
+                T.cast(T.Dict[str, T.Any], real_kwargs))
+
+        return ModuleReturnValue(ct, [ct])
+
+
 def initialize(interp: 'Interpreter') -> I18nModule:
     return I18nModule(interp)

mesonbuild/scripts/itstool.py

@@ -0,0 +1,82 @@
+# Copyright 2016 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import argparse
+import subprocess
+import tempfile
+import shutil
+import typing as T
+
+parser = argparse.ArgumentParser()
+parser.add_argument('command')
+parser.add_argument('--build-dir', default='')
+parser.add_argument('-i', '--input', default='')
+parser.add_argument('-o', '--output', default='')
+parser.add_argument('--its', action='append', default=[])
+parser.add_argument('mo_files', nargs='+')
+
+
+def run_join(build_dir: str, its_files: T.List[str], mo_files: T.List[str], in_fname: str, out_fname: str) -> int:
+    if not mo_files:
+        print('No mo files specified to use for translation.')
+        return 1
+
+    with tempfile.TemporaryDirectory(prefix=os.path.basename(in_fname), dir=build_dir) as tmp_dir:
+        # copy mo files to have the right names so itstool can infer their locale
+        locale_mo_files = []
+        for mo_file in mo_files:
+            if not os.path.exists(mo_file):
+                print('Could not find mo file {}'.format(mo_file))
+                return 1
+            if not mo_file.endswith('.mo'):
+                print('File is not a mo file: {}'.format(mo_file))
+                return 1
+            # determine locale of this mo file
+            parts = mo_file.partition('LC_MESSAGES')
+            if parts[0].endswith((os.sep, '/')):
+                locale = os.path.basename(parts[0][:-1])
+            else:
+                locale = os.path.basename(parts[0])
+            tmp_mo_fname = os.path.join(tmp_dir, locale + '.mo')
+            shutil.copy(mo_file, tmp_mo_fname)
+            locale_mo_files.append(tmp_mo_fname)
+
+        cmd = ['itstool']
+        if its_files:
+            for fname in its_files:
+                cmd.extend(['-i', fname])
+        cmd.extend(['-j', in_fname,
+                    '-o', out_fname])
+        cmd.extend(locale_mo_files)
+
+        return subprocess.call(cmd)
+
+
+def run(args: T.List[str]) -> int:
+    options = parser.parse_args(args)
+    command = options.command
+    build_dir = os.environ.get('MESON_BUILD_ROOT', os.getcwd())
+    if options.build_dir:
+        build_dir = options.build_dir
+
+    if command == 'join':
+        return run_join(build_dir,
+                        options.its,
+                        options.mo_files,
+                        options.input,
+                        options.output)
+    else:
+        print('Unknown subcommand.')
+        return 1

test cases/frameworks/6 gettext/data3/com.mesonbuild.test.intlprog.metainfo.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<component type="console-application">
+  <id>com.mesonbuild.test.intlprog</id>
+
+  <name>Test</name>
+  <summary>Application</summary>
+
+  <metadata_license>FSFAP</metadata_license>
+  <project_license>FSFAP</project_license>
+
+  <description>
+    <p>
+      Test Application
+    </p>
+    <p>
+      International greeting.
+    </p>
+    <p>
+      This is <code>text</code> with <em>embedded XML tags</em>. Nice!
+    </p>
+  </description>
+
+  <icon type="stock">meson-unittest-intlprog</icon>
+
+  <categories>
+    <category>Development</category>
+    <category>Building</category>
+  </categories>
+
+  <provides>
+    <binary>intlprog</binary>
+  </provides>
+</component>

test cases/frameworks/6 gettext/data3/meson.build

@@ -0,0 +1,33 @@
+
+if itstool.found()
+
+  mi_translated = i18n.itstool_join(
+    input:  'com.mesonbuild.test.intlprog.metainfo.xml',
+    output: 'com.mesonbuild.test.intlprog.metainfo.xml',
+    mo_targets: mo_targets,
+    its_files: ['metainfo.its'],
+    install: true,
+    install_dir: get_option('datadir') / 'metainfo',
+  )
+
+  # older versions of itstool have a bug where ITS rules specified on the command-line
+  # are not read when joining files. Since we don't install appstream in the Meson CI
+  # environment, the to-be-tested entry will be untranslated and the test would fail, so
+  # we just skip verification if the installed itstool is too old.
+  r = run_command(itstool, '-v', check: true)
+  itstool_v = r.stdout().strip().split()
+  if itstool_v[1].version_compare('>=2.0.6')
+    verify_exe = find_program('verify.py')
+    test('test xml translation',
+        verify_exe,
+        args: [mi_translated,
+              '<p xml:lang="de">Dies ist <code>Text</code> mit <em>eingebetteten XML Tags</em>. Toll!</p>']
+    )
+  else
+    message('Skipping translation verification: Itstool too old.')
+  endif
+
+else
+  install_data('com.mesonbuild.test.intlprog.metainfo.xml',
+      install_dir: get_option('datadir') / 'metainfo')
+endif

test cases/frameworks/6 gettext/data3/metainfo.its

@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<!--
+  Copyright (C) 2015-2021 Matthias Klumpp <matthias@tenstral.net>
+  Copyright (C) 2019 Takao Fujiwara <takao.fujiwara1@gmail.com>
+  SPDX-License-Identifier: FSFAP
+-->
+<its:rules xmlns:its="http://www.w3.org/2005/11/its"
+           version="2.0">
+
+  <its:withinTextRule withinText="yes" selector="/component//description//em   |
+                                                 /component//description//code"/>
+
+  <its:translateRule selector="/component" translate="no"/>
+  <its:translateRule selector="/component/name |
+                               /component/summary |
+                               /component/description |
+                               /component/developer_name |
+                               /component/name_variant_suffix |
+                               /component/screenshots/screenshot/caption |
+                               /component/releases/release/description |
+                               /component/agreement/agreement_section/name |
+                               /component/agreement/agreement_section/description"
+                     translate="yes"/>
+
+  <its:translateRule selector="/component/name[@translatable = 'no']"
+                     translate="no"/>
+  <its:translateRule selector="/component/developer_name[@translatable = 'no']"
+                     translate="no"/>
+  <its:translateRule selector="/component/name_variant_suffix[@translatable = 'no']"
+                     translate="no"/>
+  <its:translateRule selector="/component/releases/release/description[@translatable = 'no']"
+                     translate="no"/>
+</its:rules>

test cases/frameworks/6 gettext/data3/verify.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+assert len(sys.argv) == 3
+
+fname = sys.argv[1]
+check_str = sys.argv[2]
+
+assert os.path.isfile(fname)
+with open(fname, 'r', encoding='utf-8') as f:
+    assert check_str in f.read()

test cases/frameworks/6 gettext/meson.build

@@ -15,10 +15,13 @@ if not intl.found()
   error('MESON_SKIP_TEST libintl/gettext functions not found.')
 endif
 
+itstool = find_program('itstool', required: false)
+
 i18n = import('i18n')
 
 subdir('po')
 subdir('src')
 subdir('data')
 subdir('data2')
+subdir('data3')
 subdir('generated')

test cases/frameworks/6 gettext/po/de.po

@@ -20,3 +20,7 @@ msgstr ""
 #: src/intlmain.c:15
 msgid "International greeting."
 msgstr "Internationale Gruss."
+
+#: data/com.mesonbuild.test.dummy.metainfo.xml:19
+msgid "This is <code>text</code> with <em>embedded XML tags</em>. Nice!"
+msgstr "Dies ist <code>Text</code> mit <em>eingebetteten XML Tags</em>. Toll!"

test cases/frameworks/6 gettext/po/intltest.pot

@@ -32,3 +32,7 @@ msgstr ""
 #: data/test.desktop.in:5
 msgid "Test Application"
 msgstr ""
+
+#: data/com.mesonbuild.test.dummy.metainfo.xml:19
+msgid "This is <code>text</code> with <em>embedded XML tags</em>. Nice!"
+msgstr ""

test cases/frameworks/6 gettext/po/meson.build

@@ -1,3 +1,4 @@
 langs = ['fi', 'de', 'ru']
 
-i18n.gettext('intltest', languages : langs)
+gettext_targets = i18n.gettext('intltest', languages : langs)
+mo_targets = gettext_targets[0]

test cases/frameworks/6 gettext/test.json

@@ -11,7 +11,8 @@
     {"type": "file", "file": "usr/share/applications/test3.desktop"},
     {"type": "file", "file": "usr/share/applications/test4.desktop"},
     {"type": "file", "file": "usr/share/applications/test5.desktop"},
-    {"type": "file", "file": "usr/share/applications/test6.desktop"}
+    {"type": "file", "file": "usr/share/applications/test6.desktop"},
+    {"type": "file", "file": "usr/share/metainfo/com.mesonbuild.test.intlprog.metainfo.xml"}
   ],
   "matrix": {
     "options": {