Fix searching of shared libraries on OpenBSD (#3851)

* get_library_naming: Use templates instead of suffix/prefix pairs

This commit does not change functionality, and merely sets the
groundwork for a more flexibly naming implementation.

* find_library: Fix manual searching on OpenBSD

On OpenBSD, shared libraries are called libfoo.so.X.Y where X is the
major version and Y is the minor version. We were assuming that it's
libfoo.so and not finding shared libraries at all while doing manual
searching, which meant we'd link statically instead.

See: https://www.openbsd.org/faq/ports/specialtopics.html#SharedLibs

Now we use file globbing to do searching, and pick the first one
that's a real file.

Closes https://github.com/mesonbuild/meson/issues/3844

* find_library: Fix priority of library search in OpenBSD

Also add unit tests for the library naming function so that it's
absolutely clear what the priority list of naming is.

Testing is done with mocking on Linux to ensure that local testing
is easy

mesonbuild/backend/ninjabackend.py

@@ -12,7 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os, pickle, re, shlex, subprocess
+import os
+import re
+import shlex
+import pickle
+import subprocess
 from collections import OrderedDict
 import itertools
 from pathlib import PurePath
@@ -24,7 +28,7 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..compilers import CompilerArgs, get_macos_dylib_install_name
+from ..compilers import CompilerArgs, CCompiler, get_macos_dylib_install_name
 from ..linkers import ArLinker
 from ..mesonlib import File, MesonException, OrderedSet
 from ..mesonlib import get_compiler_for_source, has_path_sep
@@ -2476,13 +2480,18 @@ rule FORTRAN_DEP_HACK%s
         target_args = self.build_target_link_arguments(linker, target.link_whole_targets)
         return linker.get_link_whole_for(target_args) if len(target_args) else []
 
-    def guess_library_absolute_path(self, libname, search_dirs, prefixes, suffixes):
-        for directory in search_dirs:
-            for suffix in suffixes:
-                for prefix in prefixes:
-                    trial = os.path.join(directory, prefix + libname + '.' + suffix)
-                    if os.path.isfile(trial):
-                        return trial
+    @staticmethod
+    def guess_library_absolute_path(linker, libname, search_dirs, patterns):
+        for d in search_dirs:
+            for p in patterns:
+                trial = CCompiler._get_trials_from_pattern(p, d, libname)
+                if not trial:
+                    continue
+                trial = CCompiler._get_file_from_list(trial)
+                if not trial:
+                    continue
+                # Return the first result
+                return trial
 
     def guess_external_link_dependencies(self, linker, target, commands, internal):
         # Ideally the linker would generate dependency information that could be used.
@@ -2531,17 +2540,19 @@ rule FORTRAN_DEP_HACK%s
         # TODO The get_library_naming requirement currently excludes link targets that use d or fortran as their main linker
         if hasattr(linker, 'get_library_naming'):
             search_dirs = list(search_dirs) + linker.get_library_dirs()
-            prefixes_static, suffixes_static = linker.get_library_naming(self.environment, 'static', strict=True)
-            prefixes_shared, suffixes_shared = linker.get_library_naming(self.environment, 'shared', strict=True)
+            static_patterns = linker.get_library_naming(self.environment, 'static', strict=True)
+            shared_patterns = linker.get_library_naming(self.environment, 'shared', strict=True)
             for libname in libs:
                 # be conservative and record most likely shared and static resolution, because we don't know exactly
                 # which one the linker will prefer
-                static_resolution = self.guess_library_absolute_path(libname, search_dirs, prefixes_static, suffixes_static)
-                shared_resolution = self.guess_library_absolute_path(libname, search_dirs, prefixes_shared, suffixes_shared)
-                if static_resolution:
-                    guessed_dependencies.append(os.path.realpath(static_resolution))
-                if shared_resolution:
-                    guessed_dependencies.append(os.path.realpath(shared_resolution))
+                staticlibs = self.guess_library_absolute_path(linker, libname,
+                                                              search_dirs, static_patterns)
+                sharedlibs = self.guess_library_absolute_path(linker, libname,
+                                                              search_dirs, shared_patterns)
+                if staticlibs:
+                    guessed_dependencies.append(os.path.realpath(staticlibs))
+                if sharedlibs:
+                    guessed_dependencies.append(os.path.realpath(sharedlibs))
 
         return guessed_dependencies + absolute_libs
 

mesonbuild/compilers/c.py

@@ -12,14 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import subprocess, os.path, re
+import re
+import glob
+import os.path
+import subprocess
 
 from .. import mlog
 from .. import coredata
 from . import compilers
 from ..mesonlib import (
     EnvironmentException, version_compare, Popen_safe, listify,
-    for_windows, for_darwin, for_cygwin, for_haiku,
+    for_windows, for_darwin, for_cygwin, for_haiku, for_openbsd,
 )
 
 from .compilers import (
@@ -800,6 +803,22 @@ class CCompiler(Compiler):
                         return False
         raise RuntimeError('BUG: {!r} check failed unexpectedly'.format(n))
 
+    def _get_patterns(self, env, prefixes, suffixes, shared=False):
+        patterns = []
+        for p in prefixes:
+            for s in suffixes:
+                patterns.append(p + '{}.' + s)
+        if shared and for_openbsd(self.is_cross, env):
+            # Shared libraries on OpenBSD can be named libfoo.so.X.Y:
+            # https://www.openbsd.org/faq/ports/specialtopics.html#SharedLibs
+            #
+            # This globbing is probably the best matching we can do since regex
+            # is expensive. It's wrong in many edge cases, but it will match
+            # correctly-named libraries and hopefully no one on OpenBSD names
+            # their files libfoo.so.9a.7b.1.0
+            patterns.append('lib{}.so.[0-9]*.[0-9]*')
+        return patterns
+
     def get_library_naming(self, env, libtype, strict=False):
         '''
         Get library prefixes and suffixes for the target platform ordered by
@@ -830,18 +849,37 @@ class CCompiler(Compiler):
         else:
             # Linux/BSDs
             shlibext = ['so']
+        patterns = []
         # Search priority
         if libtype in ('default', 'shared-static'):
-            suffixes = shlibext + stlibext
+            patterns += self._get_patterns(env, prefixes, shlibext, True)
+            patterns += self._get_patterns(env, prefixes, stlibext, False)
         elif libtype == 'static-shared':
-            suffixes = stlibext + shlibext
+            patterns += self._get_patterns(env, prefixes, stlibext, False)
+            patterns += self._get_patterns(env, prefixes, shlibext, True)
         elif libtype == 'shared':
-            suffixes = shlibext
+            patterns += self._get_patterns(env, prefixes, shlibext, True)
         elif libtype == 'static':
-            suffixes = stlibext
+            patterns += self._get_patterns(env, prefixes, stlibext, False)
         else:
             raise AssertionError('BUG: unknown libtype {!r}'.format(libtype))
-        return prefixes, suffixes
+        return patterns
+
+    @staticmethod
+    def _get_trials_from_pattern(pattern, directory, libname):
+        f = os.path.join(directory, pattern.format(libname))
+        if '*' in pattern:
+            # NOTE: globbing matches directories and broken symlinks
+            # so we have to do an isfile test on it later
+            return glob.glob(f)
+        return [f]
+
+    @staticmethod
+    def _get_file_from_list(files):
+        for f in files:
+            if os.path.isfile(f):
+                return f
+        return None
 
     def find_library_real(self, libname, env, extra_dirs, code, libtype):
         # First try if we can just add the library as -l.
@@ -851,36 +889,38 @@ class CCompiler(Compiler):
             args = ['-l' + libname]
             if self.links(code, env, extra_args=args):
                 return args
-        # Search in the system libraries too
-        system_dirs = self.get_library_dirs()
         # Not found or we want to use a specific libtype? Try to find the
         # library file itself.
-        prefixes, suffixes = self.get_library_naming(env, libtype)
-        # Triply-nested loops!
+        patterns = self.get_library_naming(env, libtype)
         for d in extra_dirs:
-            for suffix in suffixes:
-                for prefix in prefixes:
-                    trial = os.path.join(d, prefix + libname + '.' + suffix)
-                    if os.path.isfile(trial):
-                        return [trial]
-        for d in system_dirs:
-            for suffix in suffixes:
-                for prefix in prefixes:
-                    trial = os.path.join(d, prefix + libname + '.' + suffix)
-                    # When searching the system paths used by the compiler, we
-                    # need to check linking with link-whole, as static libs
-                    # (.a) need to be checked to ensure they are the right
-                    # architecture, e.g. 32bit or 64-bit.
-                    # Just a normal test link won't work as the .a file doesn't
-                    # seem to be checked by linker if there are no unresolved
-                    # symbols from the main C file.
-                    extra_link_args = self.get_link_whole_for([trial])
-                    extra_link_args = self.linker_to_compiler_args(extra_link_args)
-                    if (os.path.isfile(trial) and
-                            self.links(code, env,
-                                       extra_args=extra_link_args)):
-                        return [trial]
-        # XXX: For OpenBSD and macOS we (may) need to search for libfoo.x{,.y.z}.ext
+            for p in patterns:
+                trial = self._get_trials_from_pattern(p, d, libname)
+                if not trial:
+                    continue
+                trial = self._get_file_from_list(trial)
+                if not trial:
+                    continue
+                return [trial]
+        # Search in the system libraries too
+        for d in self.get_library_dirs():
+            for p in patterns:
+                trial = self._get_trials_from_pattern(p, d, libname)
+                if not trial:
+                    continue
+                trial = self._get_file_from_list(trial)
+                if not trial:
+                    continue
+                # When searching the system paths used by the compiler, we
+                # need to check linking with link-whole, as static libs
+                # (.a) need to be checked to ensure they are the right
+                # architecture, e.g. 32bit or 64-bit.
+                # Just a normal test link won't work as the .a file doesn't
+                # seem to be checked by linker if there are no unresolved
+                # symbols from the main C file.
+                extra_link_args = self.get_link_whole_for([trial])
+                extra_link_args = self.linker_to_compiler_args(extra_link_args)
+                if self.links(code, env, extra_args=extra_link_args):
+                    return [trial]
         return None
 
     def find_library_impl(self, libname, env, extra_dirs, code, libtype):

mesonbuild/compilers/fortran.py

@@ -173,8 +173,11 @@ class FortranCompiler(Compiler):
     def run(self, code, env, extra_args=None, dependencies=None):
         return CCompiler.run(self, code, env, extra_args, dependencies)
 
-    def get_library_naming(self, env, libtype, strict=False):
-        return CCompiler.get_library_naming(self, env, libtype, strict)
+    def _get_patterns(self, *args, **kwargs):
+        return CCompiler._get_patterns(self, *args, **kwargs)
+
+    def get_library_naming(self, *args, **kwargs):
+        return CCompiler.get_library_naming(self, *args, **kwargs)
 
     def find_library_real(self, *args):
         return CCompiler.find_library_real(self, *args)

mesonbuild/mesonlib.py

@@ -363,6 +363,18 @@ def for_haiku(is_cross, env):
         return env.cross_info.config['host_machine']['system'] == 'haiku'
     return False
 
+def for_openbsd(is_cross, env):
+    """
+    Host machine is OpenBSD?
+
+    Note: 'host' is the machine on which compiled binaries will run
+    """
+    if not is_cross:
+        return is_openbsd()
+    elif env.cross_info.has_host():
+        return env.cross_info.config['host_machine']['system'] == 'openbsd'
+    return False
+
 def exe_exists(arglist):
     try:
         p = subprocess.Popen(arglist, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

run_unittests.py

@@ -35,7 +35,7 @@ import mesonbuild.coredata
 import mesonbuild.modules.gnome
 from mesonbuild.interpreter import ObjectHolder
 from mesonbuild.mesonlib import (
-    is_windows, is_osx, is_cygwin, is_dragonflybsd,
+    is_windows, is_osx, is_cygwin, is_dragonflybsd, is_openbsd,
     windows_proof_rmtree, python_command, version_compare,
     grab_leading_numbers, BuildDirLock
 )
@@ -94,6 +94,25 @@ def skipIfNoPkgconfig(f):
         return f(*args, **kwargs)
     return wrapped
 
+class PatchModule:
+    '''
+    Fancy monkey-patching! Whee! Can't use mock.patch because it only
+    patches in the local namespace.
+    '''
+    def __init__(self, func, name, impl):
+        self.func = func
+        assert(isinstance(name, str))
+        self.func_name = name
+        self.old_impl = None
+        self.new_impl = impl
+
+    def __enter__(self):
+        self.old_impl = self.func
+        exec('{} = self.new_impl'.format(self.func_name))
+
+    def __exit__(self, *args):
+        exec('{} = self.old_impl'.format(self.func_name))
+
 
 class InternalTests(unittest.TestCase):
 
@@ -496,6 +515,76 @@ class InternalTests(unittest.TestCase):
         deps.add_pub_reqs([mock])
         self.assertEqual(deps.format_reqs(deps.pub_reqs), "some_name")
 
+    def _test_all_naming(self, cc, env, patterns, platform):
+        shr = patterns[platform]['shared']
+        stc = patterns[platform]['static']
+        p = cc.get_library_naming(env, 'shared')
+        self.assertEqual(p, shr)
+        p = cc.get_library_naming(env, 'static')
+        self.assertEqual(p, stc)
+        p = cc.get_library_naming(env, 'default')
+        self.assertEqual(p, shr + stc)
+        p = cc.get_library_naming(env, 'shared-static')
+        self.assertEqual(p, shr + stc)
+        p = cc.get_library_naming(env, 'static-shared')
+        self.assertEqual(p, stc + shr)
+
+    def test_find_library_patterns(self):
+        '''
+        Unit test for the library search patterns used by find_library()
+        '''
+        unix_static = ['lib{}.a', '{}.a']
+        msvc_static = ['lib{}.a', 'lib{}.lib', '{}.a', '{}.lib']
+        # This is the priority list of pattern matching for library searching
+        patterns = {'openbsd': {'shared': ['lib{}.so', '{}.so', 'lib{}.so.[0-9]*.[0-9]*'],
+                                'static': unix_static},
+                    'linux': {'shared': ['lib{}.so', '{}.so'],
+                              'static': unix_static},
+                    'darwin': {'shared': ['lib{}.dylib', '{}.dylib'],
+                               'static': unix_static},
+                    'cygwin': {'shared': ['cyg{}.dll', 'cyg{}.dll.a', 'lib{}.dll',
+                                          'lib{}.dll.a', '{}.dll', '{}.dll.a'],
+                               'static': ['cyg{}.a'] + unix_static},
+                    'windows-msvc': {'shared': ['lib{}.lib', '{}.lib'],
+                                     'static': msvc_static},
+                    'windows-mingw': {'shared': ['lib{}.dll.a', 'lib{}.lib', 'lib{}.dll',
+                                                 '{}.dll.a', '{}.lib', '{}.dll'],
+                                      'static': msvc_static}}
+        env = Environment('', '', get_fake_options(''))
+        cc = env.detect_c_compiler(False)
+        if is_osx():
+            self._test_all_naming(cc, env, patterns, 'darwin')
+        elif is_cygwin():
+            self._test_all_naming(cc, env, patterns, 'cygwin')
+        elif is_windows():
+            if cc.get_id() == 'msvc':
+                self._test_all_naming(cc, env, patterns, 'windows-msvc')
+            else:
+                self._test_all_naming(cc, env, patterns, 'windows-mingw')
+        else:
+            self._test_all_naming(cc, env, patterns, 'linux')
+            # Mock OpenBSD since we don't have tests for it
+            true = lambda x, y: True
+            if not is_openbsd():
+                with PatchModule(mesonbuild.compilers.c.for_openbsd,
+                                 'mesonbuild.compilers.c.for_openbsd', true):
+                    self._test_all_naming(cc, env, patterns, 'openbsd')
+            else:
+                self._test_all_naming(cc, env, patterns, 'openbsd')
+            with PatchModule(mesonbuild.compilers.c.for_darwin,
+                             'mesonbuild.compilers.c.for_darwin', true):
+                self._test_all_naming(cc, env, patterns, 'darwin')
+            with PatchModule(mesonbuild.compilers.c.for_cygwin,
+                             'mesonbuild.compilers.c.for_cygwin', true):
+                self._test_all_naming(cc, env, patterns, 'cygwin')
+            with PatchModule(mesonbuild.compilers.c.for_windows,
+                             'mesonbuild.compilers.c.for_windows', true):
+                self._test_all_naming(cc, env, patterns, 'windows-mingw')
+            cc.id = 'msvc'
+            with PatchModule(mesonbuild.compilers.c.for_windows,
+                             'mesonbuild.compilers.c.for_windows', true):
+                self._test_all_naming(cc, env, patterns, 'windows-msvc')
+
 
 class BasePlatformTests(unittest.TestCase):
     def setUp(self):