macos: Rewrite install_name for dependent built libraries on install

On macOS, we set the install_name for built libraries to
@rpath/libfoo.dylib, and when linking to the library, we set the RPATH
to its path in the build directory. This allows all built binaries to
be run as-is from the build directory (uninstalled).

However, on install, we have to strip all the RPATHs because they
point to the build directory, and we change the install_name of all
built libraries to the absolute path to the library. This causes the
install name in binaries to be out of date.

We now change that install name to point to the absolute path to each
built library after installation.

Fixes https://github.com/mesonbuild/meson/issues/3038
Fixes https://github.com/mesonbuild/meson/issues/3077

With this, the default workflow on macOS matches what everyone seems
to do, including Autotools and CMake. The next step is providing a way
for build files to override the install_name that is used after
installation for use with, f.ex., private libraries when combined with
the install_rpath: kwarg on targets.

mesonbuild/backend/backends.py

@@ -54,6 +54,16 @@ class InstallData:
         self.install_subdirs = []
         self.mesonintrospect = mesonintrospect
 
+class TargetInstallData:
+    def __init__(self, fname, outdir, aliases, strip, install_name_mappings, install_rpath, install_mode):
+        self.fname = fname
+        self.outdir = outdir
+        self.aliases = aliases
+        self.strip = strip
+        self.install_name_mappings = install_name_mappings
+        self.install_rpath = install_rpath
+        self.install_mode = install_mode
+
 class ExecutableSerialisation:
     def __init__(self, name, fname, cmd_args, env, is_cross, exe_wrapper,
                  workdir, extra_paths, capture):

mesonbuild/backend/ninjabackend.py

@@ -24,11 +24,11 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..compilers import CompilerArgs
+from ..compilers import CompilerArgs, get_macos_dylib_install_name
 from ..linkers import ArLinker
 from ..mesonlib import File, MesonException, OrderedSet
 from ..mesonlib import get_compiler_for_source, has_path_sep
-from .backends import CleanTrees, InstallData
+from .backends import CleanTrees, InstallData, TargetInstallData
 from ..build import InvalidArguments
 
 if mesonlib.is_windows():
@@ -699,10 +699,7 @@ int dummy;
         with open(install_data_file, 'wb') as ofile:
             pickle.dump(d, ofile)
 
-    def generate_target_install(self, d):
-        for t in self.build.get_targets().values():
-            if not t.should_install():
-                continue
+    def get_target_install_dirs(self, t):
         # Find the installation directory.
         if isinstance(t, build.SharedModule):
             default_install_dir = self.environment.get_shared_module_dir()
@@ -727,6 +724,36 @@ int dummy;
         else:
             custom_install_dir = False
             outdirs[0] = default_install_dir
+        return outdirs, custom_install_dir
+
+    def get_target_link_deps_mappings(self, t, prefix):
+        '''
+        On macOS, we need to change the install names of all built libraries
+        that a target depends on using install_name_tool so that the target
+        continues to work after installation. For this, we need a dictionary
+        mapping of the install_name value to the new one, so we can change them
+        on install.
+        '''
+        result = {}
+        if isinstance(t, build.StaticLibrary):
+            return result
+        for ld in t.get_all_link_deps():
+            if ld is t or not isinstance(ld, build.SharedLibrary):
+                continue
+            old = get_macos_dylib_install_name(ld.prefix, ld.name, ld.suffix, ld.soversion)
+            if old in result:
+                continue
+            fname = ld.get_filename()
+            outdirs, _ = self.get_target_install_dirs(ld)
+            new = os.path.join(prefix, outdirs[0], fname)
+            result.update({old: new})
+        return result
+
+    def generate_target_install(self, d):
+        for t in self.build.get_targets().values():
+            if not t.should_install():
+                continue
+            outdirs, custom_install_dir = self.get_target_install_dirs(t)
             # Sanity-check the outputs and install_dirs
             num_outdirs, num_out = len(outdirs), len(t.get_outputs())
             if num_outdirs != 1 and num_outdirs != num_out:
@@ -741,8 +768,10 @@ int dummy;
                 # Install primary build output (library/executable/jar, etc)
                 # Done separately because of strip/aliases/rpath
                 if outdirs[0] is not False:
-                    i = [self.get_target_filename(t), outdirs[0],
-                         t.get_aliases(), should_strip, t.install_rpath, install_mode]
+                    mappings = self.get_target_link_deps_mappings(t, d.prefix)
+                    i = TargetInstallData(self.get_target_filename(t), outdirs[0],
+                                          t.get_aliases(), should_strip, mappings,
+                                          t.install_rpath, install_mode)
                     d.targets.append(i)
                     # On toolchains/platforms that use an import library for
                     # linking (separate from the shared library with all the
@@ -756,11 +785,8 @@ int dummy;
                         else:
                             implib_install_dir = self.environment.get_import_lib_dir()
                         # Install the import library.
-                        i = [self.get_target_filename_for_linking(t),
-                             implib_install_dir,
-                             # It has no aliases, should not be stripped, and
-                             # doesn't have an install_rpath
-                             {}, False, '', install_mode]
+                        i = TargetInstallData(self.get_target_filename_for_linking(t),
+                                              implib_install_dir, {}, False, {}, '', install_mode)
                         d.targets.append(i)
                 # Install secondary outputs. Only used for Vala right now.
                 if num_outdirs > 1:
@@ -769,7 +795,8 @@ int dummy;
                         if outdir is False:
                             continue
                         f = os.path.join(self.get_target_dir(t), output)
-                        d.targets.append([f, outdir, {}, False, None, install_mode])
+                        i = TargetInstallData(f, outdir, {}, False, {}, None, install_mode)
+                        d.targets.append(i)
             elif isinstance(t, build.CustomTarget):
                 # If only one install_dir is specified, assume that all
                 # outputs will be installed into it. This is for
@@ -781,14 +808,16 @@ int dummy;
                 if num_outdirs == 1 and num_out > 1:
                     for output in t.get_outputs():
                         f = os.path.join(self.get_target_dir(t), output)
-                        d.targets.append([f, outdirs[0], {}, False, None, install_mode])
+                        i = TargetInstallData(f, outdirs[0], {}, False, {}, None, install_mode)
+                        d.targets.append(i)
                 else:
                     for output, outdir in zip(t.get_outputs(), outdirs):
                         # User requested that we not install this output
                         if outdir is False:
                             continue
                         f = os.path.join(self.get_target_dir(t), output)
-                        d.targets.append([f, outdir, {}, False, None, install_mode])
+                        i = TargetInstallData(f, outdir, {}, False, {}, None, install_mode)
+                        d.targets.append(i)
 
     def generate_custom_install_script(self, d):
         result = []
@@ -2398,7 +2427,6 @@ rule FORTRAN_DEP_HACK%s
         return linker.get_no_stdlib_link_args()
 
     def get_target_type_link_args(self, target, linker):
-        abspath = os.path.join(self.environment.get_build_dir(), target.subdir)
         commands = []
         if isinstance(target, build.Executable):
             # Currently only used with the Swift compiler to add '-emit-executable'
@@ -2422,8 +2450,7 @@ rule FORTRAN_DEP_HACK%s
             commands += linker.get_pic_args()
             # Add -Wl,-soname arguments on Linux, -install_name on OS X
             commands += linker.get_soname_args(target.prefix, target.name, target.suffix,
-                                               abspath, target.soversion,
-                                               isinstance(target, build.SharedModule))
+                                               target.soversion, isinstance(target, build.SharedModule))
             # This is only visited when building for Windows using either GCC or Visual Studio
             if target.vs_module_defs and hasattr(linker, 'gen_vs_module_defs_args'):
                 commands += linker.gen_vs_module_defs_args(target.vs_module_defs.rel_to_builddir(self.build_to_src))

mesonbuild/compilers/__init__.py

@@ -30,6 +30,7 @@ __all__ = [
     'clike_langs',
     'c_suffixes',
     'cpp_suffixes',
+    'get_macos_dylib_install_name',
     'get_base_compile_args',
     'get_base_link_args',
     'is_assembly',
@@ -105,6 +106,7 @@ from .compilers import (
     clike_langs,
     c_suffixes,
     cpp_suffixes,
+    get_macos_dylib_install_name,
     get_base_compile_args,
     get_base_link_args,
     is_header,

mesonbuild/compilers/c.py

@@ -93,7 +93,7 @@ class CCompiler(Compiler):
         # Almost every compiler uses this for disabling warnings
         return ['-w']
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, *args):
         return []
 
     def split_shlib_to_parts(self, fname):

mesonbuild/compilers/compilers.py

@@ -970,7 +970,9 @@ class Compiler:
         abs_rpaths = [os.path.join(build_dir, p) for p in rpath_paths]
         if build_rpath != '':
             abs_rpaths.append(build_rpath)
-        args = ['-Wl,-rpath,' + rp for rp in abs_rpaths]
+        # Ensure that there is enough space for large RPATHs
+        args = ['-Wl,-headerpad_max_install_names']
+        args += ['-Wl,-rpath,' + rp for rp in abs_rpaths]
         return args
 
     def build_unix_rpath_args(self, build_dir, from_dir, rpath_paths, build_rpath, install_rpath):
@@ -1056,7 +1058,14 @@ ICC_WIN = 2
 GNU_LD_AS_NEEDED = '-Wl,--as-needed'
 APPLE_LD_AS_NEEDED = '-Wl,-dead_strip_dylibs'
 
-def get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+def get_macos_dylib_install_name(prefix, shlib_name, suffix, soversion):
+    install_name = prefix + shlib_name
+    if soversion is not None:
+        install_name += '.' + soversion
+    install_name += '.dylib'
+    return '@rpath/' + install_name
+
+def get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, soversion, is_shared_module):
     if soversion is None:
         sostr = ''
     else:
@@ -1069,11 +1078,8 @@ def get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, path, soversion, i
     elif gcc_type == GCC_OSX:
         if is_shared_module:
             return []
-        install_name = prefix + shlib_name
-        if soversion is not None:
-            install_name += '.' + soversion
-        install_name += '.dylib'
-        return ['-install_name', os.path.join('@rpath', install_name)]
+        name = get_macos_dylib_install_name(prefix, shlib_name, suffix, soversion)
+        return ['-install_name', name]
     else:
         raise RuntimeError('Not implemented yet.')
 
@@ -1213,8 +1219,8 @@ class GnuCompiler:
     def split_shlib_to_parts(self, fname):
         return os.path.dirname(fname), fname
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
-        return get_gcc_soname_args(self.gcc_type, prefix, shlib_name, suffix, path, soversion, is_shared_module)
+    def get_soname_args(self, prefix, shlib_name, suffix, soversion, is_shared_module):
+        return get_gcc_soname_args(self.gcc_type, prefix, shlib_name, suffix, soversion, is_shared_module)
 
     def get_std_shared_lib_link_args(self):
         return ['-shared']
@@ -1330,7 +1336,7 @@ class ClangCompiler:
         # so it might change semantics at any time.
         return ['-include-pch', os.path.join(pch_dir, self.get_pch_name(header))]
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, prefix, shlib_name, suffix, soversion, is_shared_module):
         if self.clang_type == CLANG_STANDARD:
             gcc_type = GCC_STANDARD
         elif self.clang_type == CLANG_OSX:
@@ -1339,7 +1345,7 @@ class ClangCompiler:
             gcc_type = GCC_MINGW
         else:
             raise MesonException('Unreachable code when converting clang type to gcc type.')
-        return get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, path, soversion, is_shared_module)
+        return get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, soversion, is_shared_module)
 
     def has_multi_arguments(self, args, env):
         myargs = ['-Werror=unknown-warning-option', '-Werror=unused-command-line-argument']
@@ -1422,7 +1428,7 @@ class IntelCompiler:
     def split_shlib_to_parts(self, fname):
         return os.path.dirname(fname), fname
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, prefix, shlib_name, suffix, soversion, is_shared_module):
         if self.icc_type == ICC_STANDARD:
             gcc_type = GCC_STANDARD
         elif self.icc_type == ICC_OSX:
@@ -1431,7 +1437,7 @@ class IntelCompiler:
             gcc_type = GCC_MINGW
         else:
             raise MesonException('Unreachable code when converting icc type to gcc type.')
-        return get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, path, soversion, is_shared_module)
+        return get_gcc_soname_args(gcc_type, prefix, shlib_name, suffix, soversion, is_shared_module)
 
     # TODO: centralise this policy more globally, instead
     # of fragmenting it into GnuCompiler and ClangCompiler

mesonbuild/compilers/cs.py

@@ -41,7 +41,7 @@ class CsCompiler(Compiler):
     def get_link_args(self, fname):
         return ['-r:' + fname]
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, *args):
         return []
 
     def get_werror_args(self):

mesonbuild/compilers/d.py

@@ -89,9 +89,9 @@ class DCompiler(Compiler):
     def get_std_shared_lib_link_args(self):
         return ['-shared']
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, prefix, shlib_name, suffix, soversion, is_shared_module):
         # FIXME: Make this work for Windows, MacOS and cross-compiling
-        return get_gcc_soname_args(GCC_STANDARD, prefix, shlib_name, suffix, path, soversion, is_shared_module)
+        return get_gcc_soname_args(GCC_STANDARD, prefix, shlib_name, suffix, soversion, is_shared_module)
 
     def get_feature_args(self, kwargs, build_to_src):
         res = []

mesonbuild/compilers/fortran.py

@@ -58,14 +58,14 @@ class FortranCompiler(Compiler):
     def get_no_warn_args(self):
         return CCompiler.get_no_warn_args(self)
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
-        return CCompiler.get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module)
+    def get_soname_args(self, *args):
+        return CCompiler.get_soname_args(self, *args)
 
     def split_shlib_to_parts(self, fname):
         return CCompiler.split_shlib_to_parts(self, fname)
 
-    def build_rpath_args(self, build_dir, from_dir, rpath_paths, build_rpath, install_rpath):
-        return CCompiler.build_rpath_args(self, build_dir, from_dir, rpath_paths, build_rpath, install_rpath)
+    def build_rpath_args(self, *args):
+        return CCompiler.build_rpath_args(self, *args)
 
     def get_dependency_gen_args(self, outtarget, outfile):
         return []

mesonbuild/compilers/java.py

@@ -25,7 +25,7 @@ class JavaCompiler(Compiler):
         self.id = 'unknown'
         self.javarunner = 'java'
 
-    def get_soname_args(self, prefix, shlib_name, suffix, path, soversion, is_shared_module):
+    def get_soname_args(self, *args):
         return []
 
     def get_werror_args(self):

mesonbuild/mintro.py

@@ -53,14 +53,12 @@ def buildparser():
 def determine_installed_path(target, installdata):
     install_target = None
     for i in installdata.targets:
-        if os.path.basename(i[0]) == target.get_filename(): # FIXME, might clash due to subprojects.
+        if os.path.basename(i.fname) == target.get_filename(): # FIXME, might clash due to subprojects.
             install_target = i
             break
     if install_target is None:
         raise RuntimeError('Something weird happened. File a bug.')
-    fname = i[0]
-    outdir = i[1]
-    outname = os.path.join(installdata.prefix, outdir, os.path.basename(fname))
+    outname = os.path.join(installdata.prefix, i.outdir, os.path.basename(i.fname))
     # Normalize the path by using os.path.sep consistently, etc.
     # Does not change the effective path.
     return str(pathlib.PurePath(outname))
@@ -69,8 +67,9 @@ def determine_installed_path(target, installdata):
 def list_installed(installdata):
     res = {}
     if installdata is not None:
-        for path, installdir, aliases, *unknown in installdata.targets:
-            res[os.path.join(installdata.build_dir, path)] = os.path.join(installdata.prefix, installdir, os.path.basename(path))
+        for t in installdata.targets:
+            res[os.path.join(installdata.build_dir, t.fname)] = \
+                os.path.join(installdata.prefix, t.outdir, os.path.basename(t.fname))
         for path, installpath, unused_prefix in installdata.data:
             res[path] = os.path.join(installdata.prefix, installpath)
         for path, installdir, unused_custom_install_mode in installdata.headers:

mesonbuild/scripts/depfixer.py

@@ -364,7 +364,7 @@ def get_darwin_rpaths_to_remove(fname):
             result.append(rp)
     return result
 
-def fix_darwin(fname, new_rpath, final_path):
+def fix_darwin(fname, new_rpath, final_path, install_name_mappings):
     try:
         rpaths = get_darwin_rpaths_to_remove(fname)
     except subprocess.CalledProcessError:
@@ -385,6 +385,9 @@ def fix_darwin(fname, new_rpath, final_path):
         # Rewrite -install_name @rpath/libfoo.dylib to /path/to/libfoo.dylib
         if fname.endswith('dylib'):
             args += ['-id', final_path]
+        if install_name_mappings:
+            for old, new in install_name_mappings.items():
+                args += ['-change', old, new]
         if args:
             subprocess.check_call(['install_name_tool', fname] + args,
                                   stdout=subprocess.DEVNULL,
@@ -393,7 +396,7 @@ def fix_darwin(fname, new_rpath, final_path):
         raise
         sys.exit(0)
 
-def fix_rpath(fname, new_rpath, final_path, verbose=True):
+def fix_rpath(fname, new_rpath, final_path, install_name_mappings, verbose=True):
     # Static libraries never have rpaths
     if fname.endswith('.a'):
         return

mesonbuild/scripts/meson_install.py

@@ -360,14 +360,16 @@ def check_for_stampfile(fname):
 
 def install_targets(d):
     for t in d.targets:
-        fname = check_for_stampfile(t[0])
-        outdir = get_destdir_path(d, t[1])
+        fname = check_for_stampfile(t.fname)
+        outdir = get_destdir_path(d, t.outdir)
+        final_path = os.path.join(d.prefix, t.outdir, fname)
         outname = os.path.join(outdir, os.path.basename(fname))
         final_path = os.path.join(d.prefix, outname)
-        aliases = t[2]
-        should_strip = t[3]
-        install_rpath = t[4]
-        install_mode = t[5]
+        aliases = t.aliases
+        should_strip = t.strip
+        install_name_mappings = t.install_name_mappings
+        install_rpath = t.install_rpath
+        install_mode = t.install_mode
         print('Installing %s to %s' % (fname, outname))
         d.dirmaker.makedirs(outdir, exist_ok=True)
         if not os.path.exists(fname):
@@ -416,7 +418,7 @@ def install_targets(d):
         if os.path.isfile(outname):
             try:
                 depfixer.fix_rpath(outname, install_rpath, final_path,
-                                   verbose=False)
+                                   install_name_mappings, verbose=False)
             except SystemExit as e:
                 if isinstance(e.code, int) and e.code == 0:
                     pass

run_unittests.py

@@ -2759,7 +2759,8 @@ class LinuxlikeTests(BasePlatformTests):
         self._test_soname_impl(self.builddir, False)
 
     def test_installed_soname(self):
-        self._test_soname_impl(self.installdir + self.libdir, True)
+        libdir = self.installdir + os.path.join(self.prefix, self.libdir)
+        self._test_soname_impl(libdir, True)
 
     def test_compiler_check_flags_order(self):
         '''
@@ -3333,33 +3334,45 @@ endian = 'little'
         self.run_tests()
 
     @skipIfNoPkgconfig
-    def test_uninstalled_usage_external_library(self):
+    def test_usage_external_library(self):
         '''
         Test that uninstalled usage of an external library (from the system or
-        PkgConfigDependency) works. On Linux/BSD/macOS it tests if RPATHs are
-        set correctly.
-
-        TODO: On Windows, this can test whether PATH is set properly
+        PkgConfigDependency) works. On macOS, this workflow works out of the
+        box. On Linux, BSDs, Windows, etc, you need to set extra arguments such
+        as LD_LIBRARY_PATH, etc, so this test is skipped.
 
         The system library is found with cc.find_library() and pkg-config deps.
         '''
+        if not is_osx():
+            raise unittest.SkipTest('workflow currently only works on macOS')
         oldprefix = self.prefix
         # Install external library so we can find it
         testdir = os.path.join(self.unit_test_dir, '33 external, internal library rpath', 'external library')
+        # install into installdir without using DESTDIR
         installdir = self.installdir
         self.prefix = installdir
         self.init(testdir)
+        self.prefix = oldprefix
         self.build()
         self.install(use_destdir=False)
-        self.prefix = oldprefix
         # New builddir for the consumer
         self.new_builddir()
         os.environ['LIBRARY_PATH'] = os.path.join(installdir, self.libdir)
         os.environ['PKG_CONFIG_PATH'] = os.path.join(installdir, self.libdir, 'pkgconfig')
         testdir = os.path.join(self.unit_test_dir, '33 external, internal library rpath', 'built library')
+        # install into installdir without using DESTDIR
+        self.prefix = self.installdir
         self.init(testdir)
+        self.prefix = oldprefix
         self.build()
+        # test uninstalled
         self.run_tests()
+        # test running after installation
+        self.install(use_destdir=False)
+        prog = os.path.join(self.installdir, 'bin', 'prog')
+        self._run([prog])
+        out = self._run(['otool', '-L', prog])
+        self.assertNotIn('@rpath', out)
 
 
 class LinuxArmCrossCompileTests(BasePlatformTests):

test cases/unit/33 external, internal library rpath/built library/meson.build

@@ -5,7 +5,8 @@ foo_system_dep = cc.find_library('foo_in_system')
 faa_pkg_dep = dependency('faa_pkg')
 
 l = shared_library('bar_built', 'bar.c',
+                   install: true,
                    dependencies : [foo_system_dep, faa_pkg_dep])
 
-e = executable('prog', 'prog.c', link_with: l)
+e = executable('prog', 'prog.c', link_with: l, install: true)
 test('testprog', e)