Merge pull request #2397 from mesonbuild/prebuilt

Better support for prebuilt shared libs

docs/markdown/snippets/prebuilt.md

@@ -0,0 +1,20 @@
+# Better support for shared libraries in non-system paths
+
+Meson has had support for prebuilt object files and static libraries.
+This release adds feature parity to shared libraries that are either
+in non-standard system paths or shipped as part of your project. On
+systems that support rpath, Meson automatically adds rpath entries
+to built targets using manually found external libraries.
+
+This means that e.g. supporting prebuilt libraries shipped with your
+source or provided by subprojects or wrap definitions by writing a
+build file like this:
+
+    project('myprebuiltlibrary', 'c')
+    
+    cc = meson.get_compiler('c')
+    prebuilt = cc.find_library('mylib', dirs : meson.current_source_dir())
+    mydep = declare_dependency(include_directories : include_directories('.'),
+                               dependencies : prebuilt)
+
+Then you can use the dependency object in the same way as any other.

mesonbuild/backend/backends.py

@@ -307,6 +307,25 @@ class Backend:
             raise MesonException(m.format(target.name))
         return l
 
+    def rpaths_for_bundled_shared_libraries(self, target):
+        paths = []
+        for dep in target.external_deps:
+            if isinstance(dep, dependencies.ExternalLibrary):
+                la = dep.link_args
+                if len(la) == 1 and os.path.isabs(la[0]):
+                    # The only link argument is an absolute path to a library file.
+                    libpath = la[0]
+                    if libpath.startswith(('/usr/lib', '/lib')):
+                        # No point in adding system paths.
+                        continue
+                    if os.path.splitext(libpath)[1] not in ['.dll', '.lib', '.so']:
+                        continue
+                    absdir = os.path.split(libpath)[0]
+                    rel_to_src = absdir[len(self.environment.get_source_dir()) + 1:]
+                    assert(not os.path.isabs(rel_to_src))
+                    paths.append(os.path.join(self.build_to_src, rel_to_src))
+        return paths
+
     def determine_rpath_dirs(self, target):
         link_deps = target.get_all_link_deps()
         result = []
@@ -314,6 +333,9 @@ class Backend:
             prospective = self.get_target_dir(ld)
             if prospective not in result:
                 result.append(prospective)
+        for rp in self.rpaths_for_bundled_shared_libraries(target):
+            if rp not in result:
+                result += [rp]
         return result
 
     def object_filename_from_source(self, target, source, is_unity):
@@ -522,6 +544,8 @@ class Backend:
             dirseg = os.path.join(self.environment.get_build_dir(), self.get_target_dir(ld))
             if dirseg not in result:
                 result.append(dirseg)
+        for deppath in self.rpaths_for_bundled_shared_libraries(target):
+            result.append(os.path.normpath(os.path.join(self.environment.get_build_dir(), deppath)))
         return result
 
     def write_benchmark_file(self, datafile):

run_project_tests.py

@@ -124,6 +124,8 @@ print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
 do_debug = not {'MESON_PRINT_TEST_OUTPUT', 'TRAVIS', 'APPVEYOR'}.isdisjoint(os.environ)
 no_meson_log_msg = 'No meson-log.txt found.'
 
+system_compiler = None
+
 meson_command = os.path.join(os.getcwd(), 'meson')
 if not os.path.exists(meson_command):
     meson_command += '.py'
@@ -141,9 +143,6 @@ def stop_handler(signal, frame):
 signal.signal(signal.SIGINT, stop_handler)
 signal.signal(signal.SIGTERM, stop_handler)
 
-# Needed when running cross tests because we don't generate prebuilt files
-compiler = None
-
 def setup_commands(optbackend):
     global do_debug, backend, backend_flags
     global compile_commands, clean_commands, test_commands, install_commands, uninstall_commands
@@ -451,7 +450,6 @@ def detect_tests_to_run():
         ('failing-meson', 'failing', False),
         ('failing-build', 'failing build', False),
         ('failing-tests', 'failing tests', False),
-        ('prebuilt', 'prebuilt', False),
 
         ('platform-osx', 'osx', not mesonlib.is_osx()),
         ('platform-windows', 'windows', not mesonlib.is_windows() and not mesonlib.is_cygwin()),
@@ -485,7 +483,7 @@ def run_tests(all_tests, log_name_base, extra_args):
         return _run_tests(all_tests, log_name_base, extra_args)
 
 def _run_tests(all_tests, log_name_base, extra_args):
-    global stop, executor, futures
+    global stop, executor, futures, system_compiler
     xmlname = log_name_base + '.xml'
     junit_root = ET.Element('testsuites')
     conf_time = 0
@@ -532,7 +530,7 @@ def _run_tests(all_tests, log_name_base, extra_args):
             should_fail = False
             if name.startswith('failing'):
                 should_fail = name.split('failing-')[1]
-            result = executor.submit(run_test, skipped, t, extra_args, compiler, backend, backend_flags, commands, should_fail)
+            result = executor.submit(run_test, skipped, t, extra_args, system_compiler, backend, backend_flags, commands, should_fail)
             futures.append((testname, t, result))
         for (testname, t, result) in futures:
             sys.stdout.flush()
@@ -600,52 +598,6 @@ def check_format():
                 fullname = os.path.join(root, file)
                 check_file(fullname)
 
-def pbcompile(compiler, source, objectfile):
-    if compiler == 'cl':
-        cmd = [compiler, '/nologo', '/Fo' + objectfile, '/c', source]
-    else:
-        cmd = [compiler, '-c', source, '-o', objectfile]
-    subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-
-def generate_pb_object(compiler, object_suffix):
-    source = 'test cases/prebuilt/1 object/source.c'
-    objectfile = 'test cases/prebuilt/1 object/prebuilt.' + object_suffix
-    pbcompile(compiler, source, objectfile)
-    return objectfile
-
-def generate_pb_static(compiler, object_suffix, static_suffix):
-    source = 'test cases/prebuilt/2 static/libdir/best.c'
-    objectfile = 'test cases/prebuilt/2 static/libdir/best.' + object_suffix
-    stlibfile = 'test cases/prebuilt/2 static/libdir/libbest.' + static_suffix
-    pbcompile(compiler, source, objectfile)
-    if compiler == 'cl':
-        linker = ['lib', '/NOLOGO', '/OUT:' + stlibfile, objectfile]
-    else:
-        linker = ['ar', 'csr', stlibfile, objectfile]
-    subprocess.check_call(linker)
-    os.unlink(objectfile)
-    return stlibfile
-
-def generate_prebuilt():
-    global compiler
-    static_suffix = 'a'
-    if shutil.which('cl'):
-        compiler = 'cl'
-        static_suffix = 'lib'
-    elif shutil.which('cc'):
-        compiler = 'cc'
-    elif shutil.which('gcc'):
-        compiler = 'gcc'
-    else:
-        raise RuntimeError("Could not find C compiler.")
-    if mesonlib.is_windows():
-        object_suffix = 'obj'
-    else:
-        object_suffix = 'o'
-    objectfile = generate_pb_object(compiler, object_suffix)
-    stlibfile = generate_pb_static(compiler, object_suffix, static_suffix)
-    return objectfile, stlibfile
-
 def check_meson_commands_work():
     global backend, meson_command, compile_commands, test_commands, install_commands
     testdir = 'test cases/common/1 trivial'
@@ -670,6 +622,18 @@ def check_meson_commands_work():
             if pc.returncode != 0:
                 raise RuntimeError('Failed to install {!r}:\n{}\n{}'.format(testdir, e, o))
 
+
+def detect_system_compiler():
+    global system_compiler
+    if shutil.which('cl'):
+        system_compiler = 'cl'
+    elif shutil.which('cc'):
+        system_compiler = 'cc'
+    elif shutil.which('gcc'):
+        system_compiler = 'gcc'
+    else:
+        raise RuntimeError("Could not find C compiler.")
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description="Run the test suite of Meson.")
     parser.add_argument('extra_args', nargs='*',
@@ -679,19 +643,17 @@ if __name__ == '__main__':
     options = parser.parse_args()
     setup_commands(options.backend)
 
+    detect_system_compiler()
     script_dir = os.path.split(__file__)[0]
     if script_dir != '':
         os.chdir(script_dir)
     check_format()
     check_meson_commands_work()
-    pbfiles = generate_prebuilt()
     try:
         all_tests = detect_tests_to_run()
         (passing_tests, failing_tests, skipped_tests) = run_tests(all_tests, 'meson-test-run', options.extra_args)
     except StopException:
         pass
-    for f in pbfiles:
-        os.unlink(f)
     print('\nTotal passed tests:', green(str(passing_tests)))
     print('Total failed tests:', red(str(failing_tests)))
     print('Total skipped tests:', yellow(str(skipped_tests)))

run_unittests.py

@@ -1341,6 +1341,100 @@ int main(int argc, char **argv) {
         for i in targets:
             self.assertPathExists(os.path.join(testdir, i))
 
+    def detect_prebuild_env(self):
+        if mesonbuild.mesonlib.is_windows():
+            object_suffix = 'obj'
+        else:
+            object_suffix = 'o'
+        static_suffix = 'a'
+        shared_suffix = 'so'
+        if shutil.which('cl'):
+            compiler = 'cl'
+            static_suffix = 'lib'
+            shared_suffix = 'dll'
+        elif shutil.which('cc'):
+            compiler = 'cc'
+        elif shutil.which('gcc'):
+            compiler = 'gcc'
+        else:
+            raise RuntimeError("Could not find C compiler.")
+        return (compiler, object_suffix, static_suffix, shared_suffix)
+
+    def pbcompile(self, compiler, source, objectfile, extra_args=[]):
+        if compiler == 'cl':
+            cmd = [compiler, '/nologo', '/Fo' + objectfile, '/c', source] + extra_args
+        else:
+            cmd = [compiler, '-c', source, '-o', objectfile] + extra_args
+        subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+    def test_prebuilt_object(self):
+        (compiler, object_suffix, _, _) = self.detect_prebuild_env()
+        tdir = os.path.join(self.unit_test_dir, '14 prebuilt object')
+        source = os.path.join(tdir, 'source.c')
+        objectfile = os.path.join(tdir, 'prebuilt.' + object_suffix)
+        self.pbcompile(compiler, source, objectfile)
+        try:
+            self.init(tdir)
+            self.build()
+            self.run_tests()
+        finally:
+            os.unlink(objectfile)
+
+    def test_prebuilt_static_lib(self):
+        (compiler, object_suffix, static_suffix, _) = self.detect_prebuild_env()
+        tdir = os.path.join(self.unit_test_dir, '15 prebuilt static')
+        source = os.path.join(tdir, 'libdir/best.c')
+        objectfile = os.path.join(tdir, 'libdir/best.' + object_suffix)
+        stlibfile = os.path.join(tdir, 'libdir/libbest.' + static_suffix)
+        if compiler == 'cl':
+            link_cmd = ['lib', '/NOLOGO', '/OUT:' + stlibfile, objectfile]
+        else:
+            link_cmd = ['ar', 'csr', stlibfile, objectfile]
+        self.pbcompile(compiler, source, objectfile)
+        try:
+            subprocess.check_call(link_cmd)
+        finally:
+            os.unlink(objectfile)
+        try:
+            self.init(tdir)
+            self.build()
+            self.run_tests()
+        finally:
+            os.unlink(stlibfile)
+
+    def test_prebuilt_shared_lib(self):
+        (compiler, object_suffix, _, shared_suffix) = self.detect_prebuild_env()
+        tdir = os.path.join(self.unit_test_dir, '16 prebuilt shared')
+        source = os.path.join(tdir, 'alexandria.c')
+        objectfile = os.path.join(tdir, 'alexandria.' + object_suffix)
+        if compiler == 'cl':
+            extra_args = []
+            shlibfile = os.path.join(tdir, 'alexandria.' + shared_suffix)
+            link_cmd = ['link', '/NOLOGO','/DLL', '/DEBUG', '/IMPLIB:' + os.path.join(tdir, 'alexandria.lib'), '/OUT:' + shlibfile, objectfile]
+        else:
+            extra_args = ['-fPIC']
+            shlibfile = os.path.join(tdir, 'libalexandria.' + shared_suffix)
+            link_cmd = [compiler, '-shared', '-o', shlibfile, objectfile]
+            if not mesonbuild.mesonlib.is_osx():
+                link_cmd += ['-Wl,-soname=libalexandria.so']
+        self.pbcompile(compiler, source, objectfile, extra_args=extra_args)
+        try:
+            subprocess.check_call(link_cmd)
+        finally:
+            os.unlink(objectfile)
+        try:
+            self.init(tdir)
+            self.build()
+            self.run_tests()
+        finally:
+            os.unlink(shlibfile)
+            if mesonbuild.mesonlib.is_windows():
+                # Clean up all the garbage MSVC writes in the
+                # source tree.
+                for fname in glob(os.path.join(tdir, 'alexandria.*')):
+                    if os.path.splitext(fname)[1] not in ['.c', '.h']:
+                        os.unlink(fname)
 
 class FailureTests(BasePlatformTests):
     '''

test cases/unit/16 prebuilt shared/alexandria.c

@@ -0,0 +1,6 @@
+#include"alexandria.h"
+#include<stdio.h>
+
+void alexandria_visit() {
+    printf("You are surrounded by wisdom and knowledge. You feel enlightened.\n");
+}

test cases/unit/16 prebuilt shared/alexandria.h

@@ -0,0 +1,20 @@
+#pragma once
+
+/* Both funcs here for simplicity. */
+
+#if defined _WIN32 || defined __CYGWIN__
+#if defined BUILDING_DLL
+  #define DLL_PUBLIC __declspec(dllexport)
+#else
+  #define DLL_PUBLIC __declspec(dllimport)
+#endif
+#else
+  #if defined __GNUC__
+    #define DLL_PUBLIC __attribute__ ((visibility("default")))
+  #else
+    #pragma message ("Compiler does not support symbol visibility.")
+    #define DLL_PUBLIC
+  #endif
+#endif
+
+void DLL_PUBLIC alexandria_visit();

test cases/unit/16 prebuilt shared/another_visitor.c

@@ -0,0 +1,10 @@
+#include<alexandria.h>
+#include<stdio.h>
+
+int main(int argc, char **argv) {
+    printf("Ahh, another visitor. Stay a while.\n");
+    printf("You enter the library.\n\n");
+    alexandria_visit();
+    printf("\nYou decided not to stay forever.\n");
+    return 0;
+}

test cases/unit/16 prebuilt shared/meson.build

@@ -0,0 +1,14 @@
+project('prebuilt shared library', 'c')
+
+cc = meson.get_compiler('c')
+shlib = cc.find_library('alexandria', dirs : meson.current_source_dir())
+
+exe = executable('patron', 'patron.c', dependencies : shlib)
+test('visitation', exe)
+
+d = declare_dependency(dependencies : shlib)
+
+exe2 = executable('another_visitor', 'another_visitor.c',
+  dependencies : d)
+test('another', exe2)
+

test cases/unit/16 prebuilt shared/patron.c

@@ -0,0 +1,8 @@
+#include<alexandria.h>
+#include<stdio.h>
+
+int main(int argc, char **argv) {
+    printf("You are standing outside the Great Library of Alexandria.\n");
+    printf("You decide to go inside.\n\n");
+    alexandria_visit();
+}