Add b_thinlto_cache for automatically configuring incremental ThinLTO

docs/markdown/Builtin-options.md

@@ -122,7 +122,7 @@ The following options are available. Note that they may not be
 available on all platforms or with all compilers:
 
 | Option              | Default value        | Possible values                                               | Description                                                                    |
-|---------------|----------------|------------------------------------------------------------------|-------------------------------------------------------------------------------|
+|---------------------|----------------------|---------------------------------------------------------------|--------------------------------------------------------------------------------|
 | b_asneeded          | true                 | true, false                                                   | Use -Wl,--as-needed when linking                                               |
 | b_bitcode           | false                | true, false                                                   | Embed Apple bitcode, see below                                                 |
 | b_colorout          | always               | auto, always, never                                           | Use colored output                                                             |
@@ -131,6 +131,8 @@ available on all platforms or with all compilers:
 | b_lto               | false                | true, false                                                   | Use link time optimization                                                     |
 | b_lto_threads       | 0                    | Any integer*                                                  | Use multiple threads for lto. *(Added in 0.57.0)*                              |
 | b_lto_mode          | default              | default, thin                                                 | Select between lto modes, thin and default. *(Added in 0.57.0)*                |
+| b_thinlto_cache     | false                | true, false                                                   | Enable LLVM's ThinLTO cache for faster incremental builds. *(Added in 0.64.0)* |
+| b_thinlto_cache_dir | (Internal build dir) | true, false                                                   | Specify where to store ThinLTO cache objects. *(Added in 0.64.0)*              |
 | b_ndebug            | false                | true, false, if-release                                       | Disable asserts                                                                |
 | b_pch               | true                 | true, false                                                   | Use precompiled headers                                                        |
 | b_pgo               | off                  | off, generate, use                                            | Use profile guided optimization                                                |

docs/markdown/snippets/thinlto_cache.md

@@ -0,0 +1,8 @@
+## Incremental ThinLTO with `b_thinlto_cache`
+
+[Incremental ThinLTO](https://clang.llvm.org/docs/ThinLTO.html#incremental) can now be enabled by passing
+`-Db_thinlto_cache=true` during setup. The use of caching speeds up incremental builds significantly while retaining all
+the runtime performance benefits of ThinLTO.
+
+The cache location defaults to a Meson-managed directory inside the build folder, but can be customized with
+`b_thinlto_cache_dir`.

mesonbuild/backend/ninjabackend.py

@@ -3160,7 +3160,8 @@ https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47485'''))
         else:
             commands += compilers.get_base_link_args(target.get_options(),
                                                      linker,
-                                                     isinstance(target, build.SharedModule))
+                                                     isinstance(target, build.SharedModule),
+                                                     self.environment.get_build_dir())
         # Add -nostdlib if needed; can't be overridden
         commands += self.get_no_stdlib_link_args(target, linker)
         # Add things like /NOLOGO; usually can't be overridden

mesonbuild/compilers/compilers.py

@@ -281,11 +281,12 @@ clike_debug_args = {False: [],
 base_options: 'KeyedOptionDictType' = {
     OptionKey('b_pch'): coredata.UserBooleanOption('Use precompiled headers', True),
     OptionKey('b_lto'): coredata.UserBooleanOption('Use link time optimization', False),
-    OptionKey('b_lto'): coredata.UserBooleanOption('Use link time optimization', False),
     OptionKey('b_lto_threads'): coredata.UserIntegerOption('Use multiple threads for Link Time Optimization', (None, None, 0)),
     OptionKey('b_lto_mode'): coredata.UserComboOption('Select between different LTO modes.',
                                                       ['default', 'thin'],
                                                       'default'),
+    OptionKey('b_thinlto_cache'): coredata.UserBooleanOption('Use LLVM ThinLTO caching for faster incremental builds', False),
+    OptionKey('b_thinlto_cache_dir'): coredata.UserStringOption('Directory to store ThinLTO cache objects', ''),
     OptionKey('b_sanitize'): coredata.UserComboOption('Code sanitizer to use',
                                                       ['none', 'address', 'thread', 'undefined', 'memory', 'leak', 'address,undefined'],
                                                       'none'),
@@ -383,13 +384,19 @@ def get_base_compile_args(options: 'KeyedOptionDictType', compiler: 'Compiler')
     return args
 
 def get_base_link_args(options: 'KeyedOptionDictType', linker: 'Compiler',
-                       is_shared_module: bool) -> T.List[str]:
+                       is_shared_module: bool, build_dir: str) -> T.List[str]:
     args = []  # type: T.List[str]
     try:
         if options[OptionKey('b_lto')].value:
+            thinlto_cache_dir = None
+            if get_option_value(options, OptionKey('b_thinlto_cache'), False):
+                thinlto_cache_dir = get_option_value(options, OptionKey('b_thinlto_cache_dir'), '')
+                if thinlto_cache_dir == '':
+                    thinlto_cache_dir = os.path.join(build_dir, 'meson-private', 'thinlto-cache')
             args.extend(linker.get_lto_link_args(
                 threads=get_option_value(options, OptionKey('b_lto_threads'), 0),
-                mode=get_option_value(options, OptionKey('b_lto_mode'), 'default')))
+                mode=get_option_value(options, OptionKey('b_lto_mode'), 'default'),
+                thinlto_cache_dir=thinlto_cache_dir))
     except KeyError:
         pass
     try:
@@ -975,7 +982,8 @@ class Compiler(HoldableObject, metaclass=abc.ABCMeta):
     def get_lto_compile_args(self, *, threads: int = 0, mode: str = 'default') -> T.List[str]:
         return []
 
-    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default') -> T.List[str]:
+    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default',
+                          thinlto_cache_dir: T.Optional[str] = None) -> T.List[str]:
         return self.linker.get_lto_args()
 
     def sanitizer_compile_args(self, value: str) -> T.List[str]:

mesonbuild/compilers/mixins/clang.py

@@ -53,7 +53,8 @@ class ClangCompiler(GnuLikeCompiler):
         super().__init__()
         self.defines = defines or {}
         self.base_options.update(
-            {OptionKey('b_colorout'), OptionKey('b_lto_threads'), OptionKey('b_lto_mode')})
+            {OptionKey('b_colorout'), OptionKey('b_lto_threads'), OptionKey('b_lto_mode'), OptionKey('b_thinlto_cache'),
+             OptionKey('b_thinlto_cache_dir')})
 
         # TODO: this really should be part of the linker base_options, but
         # linkers don't have base_options.
@@ -163,8 +164,13 @@ class ClangCompiler(GnuLikeCompiler):
             args.extend(super().get_lto_compile_args(threads=threads))
         return args
 
-    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default') -> T.List[str]:
+    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default',
+                          thinlto_cache_dir: T.Optional[str] = None) -> T.List[str]:
         args = self.get_lto_compile_args(threads=threads, mode=mode)
+        if mode == 'thin' and thinlto_cache_dir is not None:
+            # We check for ThinLTO linker support above in get_lto_compile_args, and all of them support
+            # get_thinlto_cache_args as well
+            args.extend(self.linker.get_thinlto_cache_args(thinlto_cache_dir))
         # In clang -flto-jobs=0 means auto, and is the default if unspecified, just like in meson
         if threads > 0:
             if not mesonlib.version_compare(self.version, '>=4.0.0'):

mesonbuild/compilers/mixins/islinker.py

@@ -48,7 +48,8 @@ class BasicLinkerIsCompilerMixin(Compiler):
     def sanitizer_link_args(self, value: str) -> T.List[str]:
         return []
 
-    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default') -> T.List[str]:
+    def get_lto_link_args(self, *, threads: int = 0, mode: str = 'default',
+                          thinlto_cache_dir: T.Optional[str] = None) -> T.List[str]:
         return []
 
     def can_linker_accept_rsp(self) -> bool:

mesonbuild/linkers/linkers.py

@@ -456,6 +456,9 @@ class DynamicLinker(metaclass=abc.ABCMeta):
     def get_lto_args(self) -> T.List[str]:
         return []
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return []
+
     def sanitizer_args(self, value: str) -> T.List[str]:
         return []
 
@@ -813,6 +816,9 @@ class AppleDynamicLinker(PosixDynamicLinkerMixin, DynamicLinker):
 
         return (args, set())
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return ["-Wl,-cache_path_lto," + path]
+
 
 class GnuDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, DynamicLinker):
 
@@ -826,6 +832,9 @@ class GnuGoldDynamicLinker(GnuDynamicLinker):
 
     id = 'ld.gold'
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return ['-Wl,-plugin-opt,cache-dir=' + path]
+
 
 class GnuBFDDynamicLinker(GnuDynamicLinker):
 
@@ -836,6 +845,9 @@ class MoldDynamicLinker(GnuDynamicLinker):
 
     id = 'ld.mold'
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return ['-Wl,--thinlto-cache-dir=' + path]
+
 
 class LLVMDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, DynamicLinker):
 
@@ -862,6 +874,9 @@ class LLVMDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, Dyna
             return self._apply_prefix('--allow-shlib-undefined')
         return []
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return ['-Wl,--thinlto-cache-dir=' + path]
+
 
 class WASMDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, DynamicLinker):
 
@@ -1304,6 +1319,9 @@ class ClangClDynamicLinker(VisualStudioLikeLinkerMixin, DynamicLinker):
     def get_win_subsystem_args(self, value: str) -> T.List[str]:
         return self._apply_prefix([f'/SUBSYSTEM:{value.upper()}'])
 
+    def get_thinlto_cache_args(self, path: str) -> T.List[str]:
+        return ["/lldltocache:" + path]
+
 
 class XilinkDynamicLinker(VisualStudioLikeLinkerMixin, DynamicLinker):