ninjabackend: Try to guess library dependencies for linker invocation.

The linkers currently do not support ninja compatible output of
dependencies used while linking. Try to guess which files will be used
while linking in python code and generate conservative dependencies to
ensure changes in linked libraries are detected.

This generates dependencies on the best match for static and shared
linking, but this should not be a problem, except for spurious
rebuilding when only one of them changes, which should not be a problem.

Also makes sure to ignore any libraries generated inside the build, to
keep the optimisation working where changes in a shared library only
cause relink if the symbols have changed as well.

mesonbuild/backend/ninjabackend.py

@@ -2410,6 +2410,74 @@ rule FORTRAN_DEP_HACK
         target_args = self.build_target_link_arguments(linker, target.link_whole_targets)
         return linker.get_link_whole_for(target_args) if len(target_args) else []
 
+    def guess_library_absolute_path(self, libname, search_dirs, prefixes, suffixes):
+        for directory in search_dirs:
+            for suffix in suffixes:
+                for prefix in prefixes:
+                    trial = os.path.join(directory, prefix + libname + '.' + suffix)
+                    if os.path.isfile(trial):
+                        return trial
+
+    def guess_external_link_dependencies(self, linker, target, commands, internal):
+        # Ideally the linker would generate dependency information that could be used.
+        # But that has 2 problems:
+        # * currently ld can not create dependency information in a way that ninja can use:
+        #   https://sourceware.org/bugzilla/show_bug.cgi?id=22843
+        # * Meson optimizes libraries from the same build using the symbol extractor.
+        #   Just letting ninja use ld generated dependencies would undo this optimization.
+        search_dirs = []
+        libs = []
+        absolute_libs = []
+
+        build_dir = self.environment.get_build_dir()
+        # the following loop sometimes consumes two items from command in one pass
+        it = iter(commands)
+        for item in it:
+            if item in internal and not item.startswith('-'):
+                continue
+
+            if item.startswith('-L'):
+                if len(item) > 2:
+                    path = item[2:]
+                else:
+                    try:
+                        path = next(it)
+                    except StopIteration:
+                        mlog.warning("Generated linker command has -L argument without following path")
+                        break
+                if not os.path.isabs(path):
+                    path = os.path.join(build_dir, path)
+                search_dirs.append(path)
+            elif item.startswith('-l'):
+                if len(item) > 2:
+                    libs.append(item[2:])
+                else:
+                    try:
+                        libs.append(next(it))
+                    except StopIteration:
+                        mlog.warning("Generated linker command has '-l' argument without following library name")
+                        break
+            elif os.path.isabs(item) and self.environment.is_library(item) and os.path.isfile(item):
+                absolute_libs.append(item)
+
+        guessed_dependencies = []
+        # TODO The get_library_naming requirement currently excludes link targets that use d or fortran as their main linker
+        if hasattr(linker, 'get_library_naming'):
+            search_dirs += linker.get_library_dirs()
+            prefixes_static, suffixes_static = linker.get_library_naming(self.environment, 'static', strict=True)
+            prefixes_shared, suffixes_shared = linker.get_library_naming(self.environment, 'shared', strict=True)
+            for libname in libs:
+                # be conservative and record most likely shared and static resolution, because we don't know exactly
+                # which one the linker will prefer
+                static_resolution = self.guess_library_absolute_path(libname, search_dirs, prefixes_static, suffixes_static)
+                shared_resolution = self.guess_library_absolute_path(libname, search_dirs, prefixes_shared, suffixes_shared)
+                if static_resolution:
+                    guessed_dependencies.append(os.path.realpath(static_resolution))
+                if shared_resolution:
+                    guessed_dependencies.append(os.path.realpath(shared_resolution))
+
+        return guessed_dependencies + absolute_libs
+
     def generate_link(self, target, outfile, outname, obj_list, linker, extra_args=[]):
         if isinstance(target, build.StaticLibrary):
             linker_base = 'STATIC'
@@ -2476,7 +2544,8 @@ rule FORTRAN_DEP_HACK
             dependencies = []
         else:
             dependencies = target.get_dependencies()
-        commands += self.build_target_link_arguments(linker, dependencies)
+        internal = self.build_target_link_arguments(linker, dependencies)
+        commands += internal
         # For 'automagic' deps: Boost and GTest. Also dependency('threads').
         # pkg-config puts the thread flags itself via `Cflags:`
         for d in target.external_deps:
@@ -2500,6 +2569,10 @@ rule FORTRAN_DEP_HACK
         # symbols from those can be found here. This is needed when the
         # *_winlibs that we want to link to are static mingw64 libraries.
         commands += linker.get_option_link_args(self.environment.coredata.compiler_options)
+
+        dep_targets = []
+        dep_targets.extend(self.guess_external_link_dependencies(linker, target, commands, internal))
+
         # Set runtime-paths so we can run executables without needing to set
         # LD_LIBRARY_PATH, etc in the environment. Doesn't work on Windows.
         if has_path_sep(target.name):
@@ -2523,7 +2596,7 @@ rule FORTRAN_DEP_HACK
         # Convert from GCC-style link argument naming to the naming used by the
         # current compiler.
         commands = commands.to_native()
-        dep_targets = [self.get_dependency_filename(t) for t in dependencies]
+        dep_targets.extend([self.get_dependency_filename(t) for t in dependencies])
         dep_targets.extend([self.get_dependency_filename(t)
                             for t in target.link_depends])
         elem = NinjaBuildElement(self.all_outputs, outname, linker_rule, obj_list)

mesonbuild/compilers/c.py

@@ -754,7 +754,7 @@ class CCompiler(Compiler):
                         return False
         raise RuntimeError('BUG: {!r} check failed unexpectedly'.format(n))
 
-    def get_library_naming(self, env, libtype):
+    def get_library_naming(self, env, libtype, strict=False):
         '''
         Get library prefixes and suffixes for the target platform ordered by
         priority
@@ -762,7 +762,10 @@ class CCompiler(Compiler):
         stlibext = ['a']
         # We've always allowed libname to be both `foo` and `libfoo`,
         # and now people depend on it
-        prefixes = ['lib', '']
+        if strict and self.id != 'msvc': # lib prefix is not usually used with msvc
+            prefixes = ['lib']
+        else:
+            prefixes = ['lib', '']
         # Library suffixes and prefixes
         if for_darwin(env.is_cross_build(), env):
             shlibext = ['dylib']

run_unittests.py

@@ -1949,6 +1949,55 @@ recommended as it can lead to undefined behaviour on some platforms''')
         exe = os.path.join(self.builddir, 'main')
         self.assertEqual(b'NDEBUG=0', subprocess.check_output(exe).strip())
 
+    def test_guessed_linker_dependencies(self):
+        '''
+        Test that meson adds dependencies for libraries based on the final
+        linker command line.
+        '''
+        # build library
+        testdirbase = os.path.join(self.unit_test_dir, '26 guessed linker dependencies')
+        testdirlib = os.path.join(testdirbase, 'lib')
+        extra_args = None
+        env = Environment(testdirlib, self.builddir, self.meson_command,
+                          get_fake_options(self.prefix), [])
+        if env.detect_c_compiler(False).get_id() != 'msvc':
+            # static libraries are not linkable with -l with msvc because meson installs them
+            # as .a files which unix_args_to_native will not know as it expects libraries to use
+            # .lib as extension. For a DLL the import library is installed as .lib. Thus for msvc
+            # this tests needs to use shared libraries to test the path resolving logic in the
+            # dependency generation code path.
+            extra_args = ['--default-library', 'static']
+        self.init(testdirlib, extra_args=extra_args)
+        self.build()
+        self.install()
+        libbuilddir = self.builddir
+        installdir = self.installdir
+        libdir = os.path.join(self.installdir, self.prefix.lstrip('/').lstrip('\\'), 'lib')
+
+        # build user of library
+        self.new_builddir()
+        # replace is needed because meson mangles platform pathes passed via LDFLAGS
+        os.environ["LDFLAGS"] = '-L{}'.format(libdir.replace('\\', '/'))
+        self.init(os.path.join(testdirbase, 'exe'))
+        del os.environ["LDFLAGS"]
+        self.build()
+        self.assertBuildIsNoop()
+
+        # rebuild library
+        exebuilddir = self.builddir
+        self.installdir = installdir
+        self.builddir = libbuilddir
+        # Microsoft's compiler is quite smart about touching import libs on changes,
+        # so ensure that there is actually a change in symbols.
+        self.setconf('-Dmore_exports=true')
+        self.build()
+        self.install()
+        # no ensure_backend_detects_changes needed because self.setconf did that already
+
+        # assert user of library will be rebuild
+        self.builddir = exebuilddir
+        self.assertRebuiltTarget('app')
+
 
 class FailureTests(BasePlatformTests):
     '''

test cases/unit/26 guessed linker dependencies/exe/app.c

@@ -0,0 +1,6 @@
+void liba_func();
+
+int main() {
+    liba_func();
+    return 0;
+}

test cases/unit/26 guessed linker dependencies/exe/meson.build

@@ -0,0 +1,7 @@
+project('exe', ['c'])
+
+executable('app',
+  'app.c',
+  # Use uninterpreted strings to avoid path finding by dependency or compiler.find_library
+  link_args: ['-ltest-lib']
+  )

test cases/unit/26 guessed linker dependencies/lib/lib.c

@@ -0,0 +1,20 @@
+#if defined _WIN32
+  #define DLL_PUBLIC __declspec(dllexport)
+#else
+  #if defined __GNUC__
+    #define DLL_PUBLIC __attribute__ ((visibility("default")))
+  #else
+    #pragma message ("Compiler does not support symbol visibility.")
+    #define DLL_PUBLIC
+  #endif
+#endif
+
+void DLL_PUBLIC liba_func() {
+}
+
+#ifdef MORE_EXPORTS
+
+void DLL_PUBLIC libb_func() {
+}
+
+#endif

test cases/unit/26 guessed linker dependencies/lib/meson.build

@@ -0,0 +1,11 @@
+project('lib1', ['c'])
+
+c_args = []
+
+# Microsoft's compiler is quite smart about touching import libs on changes,
+# so ensure that there is actually a change in symbols.
+if get_option('more_exports')
+    c_args += '-DMORE_EXPORTS'
+endif
+
+a = library('test-lib', 'lib.c', c_args: c_args, install: true)

test cases/unit/26 guessed linker dependencies/lib/meson_options.txt

@@ -0,0 +1 @@
+option('more_exports', type : 'boolean', value : false)