Merge pull request #5103 from mesonbuild/linkcustom

Can link against custom targets

docs/markdown/Reference-manual.md

@@ -334,6 +334,7 @@ the following special string substitutions:
 - `@DEPFILE@` the full path to the dependency file passed to `depfile`
 - `@PLAINNAME@`: the input filename, without a path
 - `@BASENAME@`: the input filename, with extension removed
+- `@PRIVATE_DIR@`: path to a directory where the custom target must store all its intermediate files, available since 0.50.1
 
 The `depfile` keyword argument also accepts the `@BASENAME@` and `@PLAINNAME@`
 substitutions. *(since 0.47)*
@@ -519,11 +520,18 @@ be passed to [shared and static libraries](#library).
   when this file changes.
 - `link_whole` links all contents of the given static libraries
   whether they are used by not, equivalent to the
-  `-Wl,--whole-archive` argument flag of GCC, available since
-  0.40.0. As of 0.41.0 if passed a list that list will be flattened.
+  `-Wl,--whole-archive` argument flag of GCC, available since 0.40.0.
+  As of 0.41.0 if passed a list that list will be flattened. Starting
+  from version 0.51.0 this argument also accepts outputs produced by
+  custom targets. The user must ensure that the output is a library in
+  the correct format.
 - `link_with`, one or more shared or static libraries (built by this
   project) that this target should be linked with, If passed a list
-  this list will be flattened as of 0.41.0.
+  this list will be flattened as of 0.41.0. Starting with version
+  0.51.0, the arguments can also be custom targets. In this case Meson
+  will assume that merely adding the output file in the linker command
+  line is sufficient to make linking work. If this is not sufficient,
+  then the build system writer must write all other steps manually.
 - `export_dynamic` when set to true causes the target's symbols to be
   dynamically exported, allowing modules built using the
   [`shared_module`](#shared_module) function to refer to functions,

docs/markdown/snippets/linkcustom.md

@@ -0,0 +1,16 @@
+## Can link against custom targets
+
+The output of `custom_target` can be used in `link_with` and
+`link_whole` keyword arguments. This is useful for integrating custom
+code generator steps, but note that there are many limitations:
+
+ - Meson can not know about link dependencies of the custom target. If
+   the target requires further link libraries, you need to add them manually
+
+ - The user is responsible for ensuring that the code produced by
+   different toolchains are compatible.
+
+ - The custom target can only have one output file.
+
+ - The output file must have the correct file name extension.
+

mesonbuild/backend/backends.py

@@ -211,6 +211,10 @@ class Backend:
             return os.path.join(self.get_target_dir(target), link_lib)
         elif isinstance(target, build.StaticLibrary):
             return os.path.join(self.get_target_dir(target), target.get_filename())
+        elif isinstance(target, build.CustomTarget):
+            if not target.is_linkable_target():
+                raise MesonException('Tried to link against custom target "%s", which is not linkable.' % target.name)
+            return os.path.join(self.get_target_dir(target), target.get_filename())
         elif isinstance(target, build.Executable):
             if target.import_filename:
                 return os.path.join(self.get_target_dir(target), target.get_import_filename())
@@ -961,6 +965,12 @@ class Backend:
                     raise MesonException(msg)
                 dfilename = os.path.join(outdir, target.depfile)
                 i = i.replace('@DEPFILE@', dfilename)
+            elif '@PRIVATE_DIR@' in i:
+                if target.absolute_paths:
+                    pdir = self.get_target_private_dir_abs(target)
+                else:
+                    pdir = self.get_target_private_dir(target)
+                i = i.replace('@PRIVATE_DIR@', pdir)
             elif '@PRIVATE_OUTDIR_' in i:
                 match = re.search(r'@PRIVATE_OUTDIR_(ABS_)?([^/\s*]*)@', i)
                 if not match:

mesonbuild/build.py

@@ -567,6 +567,8 @@ class BuildTarget(Target):
         if self.link_targets or self.link_whole_targets:
             extra = set()
             for t in itertools.chain(self.link_targets, self.link_whole_targets):
+                if isinstance(t, CustomTarget):
+                    continue # We can't know anything about these.
                 for name, compiler in t.compilers.items():
                     if name in clink_langs:
                         extra.add((name, compiler))
@@ -1062,19 +1064,24 @@ You probably should put it in link_with instead.''')
                 msg = "Can't link non-PIC static library {!r} into shared library {!r}. ".format(t.name, self.name)
                 msg += "Use the 'pic' option to static_library to build with PIC."
                 raise InvalidArguments(msg)
-            if self.is_cross != t.is_cross:
+            if not isinstance(t, CustomTarget) and self.is_cross != t.is_cross:
                 raise InvalidArguments('Tried to mix cross built and native libraries in target {!r}'.format(self.name))
             self.link_targets.append(t)
 
     def link_whole(self, target):
         for t in listify(target, unholder=True):
-            if not isinstance(t, StaticLibrary):
+            if isinstance(t, CustomTarget):
+                if not t.is_linkable_target():
+                    raise InvalidArguments('Custom target {!r} is not linkable.'.format(t))
+                if not t.get_filename().endswith('.a'):
+                    raise InvalidArguments('Can only link_whole custom targets that are .a archives.')
+            elif not isinstance(t, StaticLibrary):
                 raise InvalidArguments('{!r} is not a static library.'.format(t))
             if isinstance(self, SharedLibrary) and not t.pic:
                 msg = "Can't link non-PIC static library {!r} into shared library {!r}. ".format(t.name, self.name)
                 msg += "Use the 'pic' option to static_library to build with PIC."
                 raise InvalidArguments(msg)
-            if self.is_cross != t.is_cross:
+            if not isinstance(t, CustomTarget) and self.is_cross != t.is_cross:
                 raise InvalidArguments('Tried to mix cross built and native libraries in target {!r}'.format(self.name))
             self.link_whole_targets.append(t)
 
@@ -1151,6 +1158,8 @@ You probably should put it in link_with instead.''')
         # Check if any of the internal libraries this target links to were
         # written in this language
         for link_target in itertools.chain(self.link_targets, self.link_whole_targets):
+            if isinstance(link_target, CustomTarget):
+                continue
             for language in link_target.compilers:
                 if language not in langs:
                     langs.append(language)
@@ -2101,6 +2110,22 @@ class CustomTarget(Target):
                 raise InvalidArguments('Substitution in depfile for custom_target that does not have an input file.')
             return self.depfile
 
+    def is_linkable_target(self):
+        if len(self.outputs) != 1:
+            return False
+        suf = os.path.splitext(self.outputs[0])[-1]
+        if suf == '.a' or suf == '.dll' or suf == '.lib' or suf == '.so':
+            return True
+
+    def get_link_deps_mapping(self, prefix, environment):
+        return {}
+
+    def get_link_dep_subdirs(self):
+        return OrderedSet()
+
+    def get_all_link_deps(self):
+        return []
+
     def type_suffix(self):
         return "@cus"
 

test cases/common/216 link custom/custom_stlib.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import shutil, sys, subprocess, argparse, pathlib
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('--private-dir', required=True)
+parser.add_argument('-o', required=True)
+parser.add_argument('cmparr', nargs='+')
+
+contents = '''#include<stdio.h>
+
+void flob() {
+    printf("Now flobbing.\\n");
+}
+'''
+
+def generate_lib_gnulike(outfile, c_file, private_dir, compiler_array):
+    if shutil.which('ar'):
+        static_linker = 'ar'
+    elif shutil.which('llvm-ar'):
+        static_linker = 'llvm-ar'
+    elif shutil.which('gcc-ar'):
+        static_linker = 'gcc-ar'
+    else:
+        sys.exit('Could not detect a static linker.')
+    o_file = c_file.with_suffix('.o')
+    compile_cmd = compiler_array + ['-c', '-g', '-O2', '-o', str(o_file), str(c_file)]
+    subprocess.check_call(compile_cmd)
+    out_file = pathlib.Path(outfile)
+    if out_file.exists():
+        out_file.unlink()
+    link_cmd = [static_linker, 'csr', outfile, str(o_file)]
+    subprocess.check_call(link_cmd)
+    return 0
+
+
+def generate_lib_msvc(outfile, c_file, private_dir, compiler_array):
+    static_linker = 'lib'
+    o_file = c_file.with_suffix('.obj')
+    compile_cmd = compiler_array + ['/MDd',
+                                    '/nologo',
+                                    '/ZI',
+                                    '/Ob0',
+                                    '/Od',
+                                    '/c',
+                                    '/Fo' + str(o_file),
+                                    str(c_file)]
+    subprocess.check_call(compile_cmd)
+    out_file = pathlib.Path(outfile)
+    if out_file.exists():
+        out_file.unlink()
+    link_cmd = [static_linker,
+                '/nologo',
+                '/OUT:' + str(outfile),
+                str(o_file)]
+    subprocess.check_call(link_cmd)
+    return 0
+
+def generate_lib(outfile, private_dir, compiler_array):
+    private_dir = pathlib.Path(private_dir)
+    if not private_dir.exists():
+        private_dir.mkdir()
+    c_file = private_dir / 'flob.c'
+    c_file.write_text(contents)
+    for i in compiler_array:
+        if (i.endswith('cl') or i.endswith('cl.exe')) and 'clang-cl' not in i:
+            return generate_lib_msvc(outfile, c_file, private_dir, compiler_array)
+    return generate_lib_gnulike(outfile, c_file, private_dir, compiler_array)
+
+if __name__ == '__main__':
+    options = parser.parse_args()
+    sys.exit(generate_lib(options.o, options.private_dir, options.cmparr))

test cases/common/216 link custom/meson.build

@@ -0,0 +1,35 @@
+project('linkcustom', 'c')
+
+# This would require passing the static linker to the build script or having
+# it detect it by itself. I'm too lazy to implement it now and it is not
+# really needed for testing that custom targets work. It is the responsibility
+# of the custom target to produce things in the correct format.
+assert(not meson.is_cross_build(),
+       'MESON_SKIP_TEST cross checking not implemented.')
+
+cc = meson.get_compiler('c')
+genprog = find_program('custom_stlib.py')
+
+clib = custom_target('linkcustom',
+  output: 'libflob.a',
+  command: [genprog,
+            '-o', '@OUTPUT@',
+            '--private-dir', '@PRIVATE_DIR@'] + cc.cmd_array())
+
+exe = executable('prog', 'prog.c', link_with: clib)
+test('linkcustom', exe)
+
+d = declare_dependency(link_with: clib)
+
+exe2 = executable('prog2', 'prog.c', dependencies: d)
+test('linkcustom2', exe2)
+
+# Link whole tests
+
+exe3 = executable('prog3', 'prog.c', link_whole: clib)
+test('linkwhole', exe)
+
+d2 = declare_dependency(link_whole: clib)
+
+exe4 = executable('prog4', 'prog.c', dependencies: d2)
+test('linkwhole2', exe2)

test cases/common/216 link custom/prog.c

@@ -0,0 +1,6 @@
+void flob();
+
+int main(int argc, char **argv) {
+    flob();
+    return 0;
+}

test cases/failing/89 link_with custom target/demo.c

@@ -1,5 +0,0 @@
-int func_in_foo();
-
-int main(int argc, char **argv) {
-    return func_in_foo();
-}

test cases/failing/89 link_with custom target/foo.c

@@ -1,3 +0,0 @@
-int func_in_foo() {
-    return 0;
-}

test cases/failing/89 link_with custom target/lib_generator.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-# Mimic a binary that generates a static library
-
-import os
-import subprocess
-import sys
-
-if __name__ == '__main__':
-    if len(sys.argv) != 4:
-        print(sys.argv[0], 'compiler input_file output_file')
-        sys.exit(1)
-    compiler = sys.argv[1]
-    ifile = sys.argv[2]
-    ofile = sys.argv[3]
-    tmp = ifile + '.o'
-    if compiler.endswith('cl'):
-        subprocess.check_call([compiler, '/nologo', '/MDd', '/Fo' + tmp, '/c', ifile])
-        subprocess.check_call(['lib', '/nologo', '/OUT:' + ofile, tmp])
-    else:
-        subprocess.check_call([compiler, '-c', ifile, '-o', tmp])
-        subprocess.check_call(['ar', 'csr', ofile, tmp])
-
-os.unlink(tmp)

test cases/failing/89 link_with custom target/meson.build

@@ -1,23 +0,0 @@
-project('link_with custom target', ['c'])
-
-#
-# libraries created by a custom_target currently can be used in sources: (see
-# common/100 manygen/ for an example of that), but not in link_with:
-#
-
-lib_generator = find_program('lib_generator.py')
-
-cc = meson.get_compiler('c').cmd_array().get(-1)
-
-libfoo_target = custom_target(
-    'libfoo',
-    input: ['foo.c'],
-    output: ['libfoo.a'],
-    command: [lib_generator, cc, '@INPUT@', '@OUTPUT@']
-)
-
-libfoo = declare_dependency(
-    link_with: libfoo_target,
-)
-
-executable('demo', ['demo.c'], dependencies: [libfoo])