New xgettext method for i18n module

This method call xgettext to extract translatable
string from source files into a .pot translation template.

It differs from a plain CustomTarget in three ways:
- It accepts build targets as sources, and automatically resolves source
  files from those build targets;
- It detects command lines that are too long, and writes, at config
  time, the list of source files into a text file to be consumed by the
  xgettext command;
- It detects dependencies between pot extraction targets, based on the
  dependencies between source targets.

docs/markdown/i18n-module.md

@@ -74,3 +74,50 @@ for normal keywords. In addition it accepts these keywords:
 * `mo_targets` *required*: mo file generation targets as returned by `i18n.gettext()`.
 
 *Added 0.62.0*
+
+
+### i18n.xgettext()
+
+``` meson
+i18n.xgettext(name, sources..., args: [...], recursive: false)
+```
+
+Invokes the `xgettext` program on given sources, to generate a `.pot` file.
+This function is to be used when the `gettext` function workflow it not suitable
+for your project. For example, it can be used to produce separate `.pot` files
+for each executable.
+
+Positional arguments are the following:
+
+* name `str`: the name of the resulting pot file.
+* sources `list[str|File|build_tgt|custom_tgt]`:
+          source files or targets. May be a list of `string`, `File`, [[@build_tgt]],
+          or [[@custom_tgt]] returned from other calls to this function.
+
+Keyword arguments are the following:
+
+- recursive `bool`:
+        if `true`, will merge the resulting pot file with extracted pot files
+        related to dependencies of the given source targets. For instance,
+        if you build an executable, then you may want to merge the executable
+        translations with the translations from the dependent libraries.
+- install `bool`: if `true`, will add the resulting pot file to install targets.
+- install_tag `str`: install tag to use for the install target.
+- install_dir `str`: directory where to install the resulting pot file.
+
+The `i18n.xgettext()` function returns a [[@custom_tgt]].
+
+Usually, you want to pass one build target as sources, and the list of header files
+for that target. If the number of source files would result in a command line that
+is too long, the list of source files is written to a file at config time, to be
+used as input for the `xgettext` program.
+
+The `recursive: true` argument is to be given to targets that will actually read
+the resulting `.mo` file. Each time you call the `i18n.xgettext()` function,
+it maps the source targets to the resulting pot file. When `recursive: true` is
+given, all generated pot files from dependencies of the source targets are
+included to generate the final pot file. Therefore, adding a dependency to
+source target will automatically add the translations of that dependency to the
+needed translations for that source target.
+
+*Added 1.8.0*

docs/markdown/snippets/i18n_xgettext.md

@@ -0,0 +1,12 @@
+## i18n module xgettext
+
+There is a new `xgettext` function in `i18n` module that acts as a
+wrapper around `xgettext`. It allows to extract strings to translate from
+source files.
+
+This function is convenient, because:
+- It can find the sources files from a build target;
+- It will use an intermediate file when the number of source files is too
+  big to be handled directly from the command line;
+- It is able to get strings to translate from the dependencies of the given
+  targets.

mesonbuild/modules/i18n.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 from os import path
+from pathlib import Path
 import shlex
 import typing as T
 
@@ -13,7 +14,8 @@ from .. import mesonlib
 from ..options import OptionKey
 from .. import mlog
 from ..interpreter.type_checking import CT_BUILD_BY_DEFAULT, CT_INPUT_KW, INSTALL_TAG_KW, OUTPUT_KW, INSTALL_DIR_KW, INSTALL_KW, NoneType, in_set_validator
-from ..interpreterbase import FeatureNew, InvalidArguments
+from ..interpreterbase import FeatureNew
+from ..interpreterbase.exceptions import InvalidArguments
 from ..interpreterbase.decorators import ContainerTypeInfo, KwargInfo, noPosargs, typed_kwargs, typed_pos_args
 from ..programs import ExternalProgram
 from ..scripts.gettext import read_linguas
@@ -65,6 +67,16 @@ if T.TYPE_CHECKING:
         its_files: T.List[str]
         mo_targets: T.List[T.Union[build.BuildTarget, build.CustomTarget, build.CustomTargetIndex]]
 
+    class XgettextProgramT(TypedDict):
+
+        args: T.List[str]
+        recursive: bool
+        install: bool
+        install_dir: T.Optional[str]
+        install_tag: T.Optional[str]
+
+    SourcesType = T.Union[str, mesonlib.File, build.BuildTarget, build.BothLibraries, build.CustomTarget]
+
 
 _ARGS: KwargInfo[T.List[str]] = KwargInfo(
     'args',
@@ -115,6 +127,125 @@ PRESET_ARGS = {
 }
 
 
+class XgettextProgram:
+
+    pot_files: T.Dict[str, build.CustomTarget] = {}
+
+    def __init__(self, xgettext: ExternalProgram, interpreter: Interpreter):
+        self.xgettext = xgettext
+        self.interpreter = interpreter
+
+    def extract(self,
+                name: str,
+                sources: T.List[SourcesType],
+                args: T.List[str],
+                recursive: bool,
+                install: bool,
+                install_dir: T.Optional[str],
+                install_tag: T.Optional[str]) -> build.CustomTarget:
+
+        if not name.endswith('.pot'):
+            name += '.pot'
+
+        source_files = self._get_source_files(sources)
+
+        command = self.xgettext.command + args
+        command.append(f'--directory={self.interpreter.environment.get_source_dir()}')
+        command.append(f'--directory={self.interpreter.environment.get_build_dir()}')
+        command.append('--output=@OUTPUT@')
+
+        depends = list(self._get_depends(sources)) if recursive else []
+        rsp_file = self._get_rsp_file(name, source_files, depends, command)
+        inputs: T.List[T.Union[mesonlib.File, build.CustomTarget]]
+        if rsp_file:
+            inputs = [rsp_file]
+            depend_files = list(source_files)
+            command.append('--files-from=@INPUT@')
+        else:
+            inputs = list(source_files) + depends
+            depends = None
+            depend_files = None
+            command.append('@INPUT@')
+
+        ct = build.CustomTarget(
+            '',
+            self.interpreter.subdir,
+            self.interpreter.subproject,
+            self.interpreter.environment,
+            command,
+            inputs,
+            [name],
+            depend_files = depend_files,
+            extra_depends = depends,
+            install = install,
+            install_dir = [install_dir] if install_dir else None,
+            install_tag = [install_tag] if install_tag else None,
+            description = 'Extracting translations to {}',
+        )
+
+        for source_id in self._get_source_id(sources):
+            self.pot_files[source_id] = ct
+        self.pot_files[ct.get_id()] = ct
+
+        self.interpreter.add_target(ct.name, ct)
+        return ct
+
+    def _get_source_files(self, sources: T.Iterable[SourcesType]) -> T.Set[mesonlib.File]:
+        source_files = set()
+        for source in sources:
+            if isinstance(source, mesonlib.File):
+                source_files.add(source)
+            elif isinstance(source, str):
+                mesonlib.check_direntry_issues(source)
+                source_files.add(mesonlib.File.from_source_file(self.interpreter.source_root, self.interpreter.subdir, source))
+            elif isinstance(source, build.BuildTarget):
+                source_files.update(source.get_sources())
+            elif isinstance(source, build.BothLibraries):
+                source_files.update(source.get('shared').get_sources())
+        return source_files
+
+    def _get_depends(self, sources: T.Iterable[SourcesType]) -> T.Set[build.CustomTarget]:
+        depends = set()
+        for source in sources:
+            if isinstance(source, build.BuildTarget):
+                for source_id in self._get_source_id(source.get_dependencies()):
+                    if source_id in self.pot_files:
+                        depends.add(self.pot_files[source_id])
+            elif isinstance(source, build.CustomTarget):
+                # Dependency on another extracted pot file
+                source_id = source.get_id()
+                if source_id in self.pot_files:
+                    depends.add(self.pot_files[source_id])
+        return depends
+
+    def _get_rsp_file(self,
+                      name: str,
+                      source_files: T.Iterable[mesonlib.File],
+                      depends: T.Iterable[build.CustomTarget],
+                      arguments: T.List[str]) -> T.Optional[mesonlib.File]:
+        source_list = '\n'.join(source.relative_name() for source in source_files)
+        for dep in depends:
+            source_list += '\n' + path.join(dep.subdir, dep.get_filename())
+
+        estimated_cmdline_length = len(source_list) + sum(len(arg) + 1 for arg in arguments) + 1
+        if estimated_cmdline_length < mesonlib.get_rsp_threshold():
+            return None
+
+        rsp_file = Path(self.interpreter.environment.build_dir, self.interpreter.subdir, name+'.rsp')
+        rsp_file.write_text(source_list, encoding='utf-8')
+
+        return mesonlib.File.from_built_file(self.interpreter.subdir, rsp_file.name)
+
+    @staticmethod
+    def _get_source_id(sources: T.Iterable[T.Union[SourcesType, build.CustomTargetIndex]]) -> T.Iterable[str]:
+        for source in sources:
+            if isinstance(source, build.Target):
+                yield source.get_id()
+            elif isinstance(source, build.BothLibraries):
+                yield source.get('static').get_id()
+                yield source.get('shared').get_id()
+
+
 class I18nModule(ExtensionModule):
 
     INFO = ModuleInfo('i18n')
@@ -125,6 +256,7 @@ class I18nModule(ExtensionModule):
             'merge_file': self.merge_file,
             'gettext': self.gettext,
             'itstool_join': self.itstool_join,
+            'xgettext': self.xgettext,
         })
         self.tools: T.Dict[str, T.Optional[T.Union[ExternalProgram, build.Executable]]] = {
             'itstool': None,
@@ -398,6 +530,27 @@ class I18nModule(ExtensionModule):
 
         return ModuleReturnValue(ct, [ct])
 
+    @FeatureNew('i18n.xgettext', '1.8.0')
+    @typed_pos_args('i18n.xgettext', str, varargs=(str, mesonlib.File, build.BuildTarget, build.BothLibraries, build.CustomTarget), min_varargs=1)
+    @typed_kwargs(
+        'i18n.xgettext',
+        _ARGS,
+        KwargInfo('recursive', bool, default=False),
+        INSTALL_KW,
+        INSTALL_DIR_KW,
+        INSTALL_TAG_KW,
+    )
+    def xgettext(self, state: ModuleState, args: T.Tuple[str, T.List[SourcesType]], kwargs: XgettextProgramT) -> build.CustomTarget:
+        toolname = 'xgettext'
+        if self.tools[toolname] is None or not self.tools[toolname].found():
+            self.tools[toolname] = state.find_program(toolname, required=True, for_machine=mesonlib.MachineChoice.BUILD)
+
+        if kwargs['install'] and not kwargs['install_dir']:
+            raise InvalidArguments('i18n.xgettext: "install_dir" keyword argument must be set when "install" is true.')
+
+        xgettext_program = XgettextProgram(T.cast('ExternalProgram', self.tools[toolname]), self.interpreter)
+        return xgettext_program.extract(*args, **kwargs)
+
 
 def initialize(interp: 'Interpreter') -> I18nModule:
     return I18nModule(interp)

test cases/frameworks/38 gettext extractor/meson.build

@@ -0,0 +1,15 @@
+project(
+    'gettext extractor',
+    'c',
+    default_options: {'default_library': 'static'},
+    meson_version: '1.8.0',
+)
+
+if not find_program('xgettext', required: false).found()
+    error('MESON_SKIP_TEST xgettext command not found')
+endif
+
+i18n = import('i18n')
+xgettext_args = ['-ktr', '--add-comments=TRANSLATOR:', '--from-code=UTF-8']
+
+subdir('src')

test cases/frameworks/38 gettext extractor/src/lib1/lib1.c

@@ -0,0 +1,10 @@
+#include "lib1.h"
+
+#include <stdio.h>
+
+#define tr(STRING) (STRING)
+
+void say_something(void)
+{
+    printf("%s\n", tr("Something!"));
+}

test cases/frameworks/38 gettext extractor/src/lib1/lib1.h

@@ -0,0 +1,6 @@
+#ifndef LIB1_H
+#define LIB1_H
+
+void say_something(void);
+
+#endif

test cases/frameworks/38 gettext extractor/src/lib1/meson.build

@@ -0,0 +1,3 @@
+lib1 = library('mylib1', 'lib1.c')
+lib1_pot = i18n.xgettext('lib1', lib1, args: xgettext_args)
+lib1_includes = include_directories('.')

test cases/frameworks/38 gettext extractor/src/lib2/lib2.c

@@ -0,0 +1,13 @@
+#include "lib2.h"
+
+#include <lib1.h>
+
+#include <stdio.h>
+
+#define tr(STRING) (STRING)
+
+void say_something_else(void)
+{
+    say_something();
+    printf("%s\n", tr("Something else!"));
+}

test cases/frameworks/38 gettext extractor/src/lib2/lib2.h

@@ -0,0 +1,6 @@
+#ifndef LIB2_H
+#define LIB2_H
+
+void say_something_else(void);
+
+#endif

test cases/frameworks/38 gettext extractor/src/lib2/meson.build

@@ -0,0 +1,3 @@
+lib2 = library('mylib2', 'lib2.c', include_directories: lib1_includes, link_with: lib1)
+lib2_pot = i18n.xgettext('lib2', lib2, args: xgettext_args)
+lib2_includes = include_directories('.')

test cases/frameworks/38 gettext extractor/src/main.c

@@ -0,0 +1,8 @@
+#include <lib2.h>
+
+int main(void)
+{
+    say_something_else();
+
+    return 0;
+}

test cases/frameworks/38 gettext extractor/src/meson.build

@@ -0,0 +1,6 @@
+subdir('lib1')
+subdir('lib2')
+
+main = executable('say', 'main.c', link_with: [lib2], include_directories: lib2_includes)
+
+main_pot = i18n.xgettext('main', main, args: xgettext_args, install: true, install_dir: 'intl', install_tag: 'intl', recursive: true)

test cases/frameworks/38 gettext extractor/test.json

@@ -0,0 +1,6 @@
+{
+    "installed": [
+        { "type": "file", "file": "usr/intl/main.pot" }
+    ],
+    "expect_skip_on_jobname": ["azure", "cygwin"]
+}