Merge pull request #696 from mesonbuild/depfile

Add support for dependency files in custom targets.

mesonbuild/backend/backends.py

@@ -565,6 +565,11 @@ class Backend():
             else:
                 if '@OUTDIR@' in i:
                     i = i.replace('@OUTDIR@', outdir)
+                elif '@DEPFILE@' in i:
+                    if target.depfile is None:
+                        raise MesonException('Custom target %s has @DEPFILE@ but no depfile keyword argument.' % target.name)
+                    dfilename = os.path.join(self.get_target_private_dir(target), target.depfile)
+                    i = i.replace('@DEPFILE@', dfilename)
                 elif '@PRIVATE_OUTDIR_' in i:
                     match = re.search('@PRIVATE_OUTDIR_(ABS_)?([-a-zA-Z0-9.@:]*)@', i)
                     source = match.group(0)
@@ -572,7 +577,6 @@ class Backend():
                         lead_dir = ''
                     else:
                         lead_dir = self.environment.get_build_dir()
-                    target_id = match.group(2)
                     i = i.replace(source,
                                   os.path.join(lead_dir,
                                                outdir))

mesonbuild/backend/ninjabackend.py

@@ -366,7 +366,11 @@ int dummy;
         desc = 'Generating {0} with a {1} command.'
         if target.build_always:
             deps.append('PHONY')
-        elem = NinjaBuildElement(self.all_outputs, ofilenames, 'CUSTOM_COMMAND', srcs)
+        if target.depfile is None:
+            rulename = 'CUSTOM_COMMAND'
+        else:
+            rulename = 'CUSTOM_COMMAND_DEP'
+        elem = NinjaBuildElement(self.all_outputs, ofilenames, rulename, srcs)
         for i in target.depend_files:
             if isinstance(i, mesonlib.File):
                 deps.append(i.rel_to_builddir(self.build_to_src))
@@ -398,6 +402,11 @@ int dummy;
         else:
             cmd_type = 'custom'
 
+        if target.depfile is not None:
+            rel_dfile = os.path.join(self.get_target_private_dir(target), target.depfile)
+            abs_pdir = os.path.join(self.environment.get_build_dir(), self.get_target_private_dir(target))
+            os.makedirs(abs_pdir, exist_ok=True)
+            elem.add_item('DEPFILE', rel_dfile)
         elem.add_item('COMMAND', cmd)
         elem.add_item('description',  desc.format(target.name, cmd_type))
         elem.write(outfile)
@@ -640,7 +649,6 @@ int dummy;
             velem.write(outfile)
 
         # And then benchmarks.
-        benchmark_script = os.path.join(script_root, 'meson_benchmark.py')
         cmd = [sys.executable, self.environment.get_build_command(), '--internal', 'benchmark', benchmark_data]
         elem = NinjaBuildElement(self.all_outputs, 'benchmark', 'CUSTOM_COMMAND', ['all', 'PHONY'])
         elem.add_item('COMMAND', cmd)
@@ -661,6 +669,14 @@ int dummy;
         outfile.write(' command = $COMMAND\n')
         outfile.write(' description = $DESC\n')
         outfile.write(' restat = 1\n\n')
+        # Ninja errors out if you have deps = gcc but no depfile, so we must
+        # have two rules for custom commands.
+        outfile.write('rule CUSTOM_COMMAND_DEP\n')
+        outfile.write(' command = $COMMAND\n')
+        outfile.write(' description = $DESC\n')
+        outfile.write(' deps = gcc\n')
+        outfile.write(' depfile = $DEPFILE\n')
+        outfile.write(' restat = 1\n\n')
         outfile.write('rule REGENERATE_BUILD\n')
         c = (quote_char + ninja_quote(sys.executable) + quote_char,
              quote_char + ninja_quote(self.environment.get_build_command())  + quote_char,
@@ -1355,8 +1371,16 @@ rule FORTRAN_DEP_HACK
             infilename = os.path.join(self.build_to_src, curfile)
             outfiles = genlist.get_outputs_for(curfile)
             outfiles = [os.path.join(self.get_target_private_dir(target), of) for of in outfiles]
+            if generator.depfile is None:
+                rulename = 'CUSTOM_COMMAND'
+                args = base_args
+            else:
+                rulename = 'CUSTOM_COMMAND_DEP'
+                depfilename = generator.get_dep_outname(infilename)
+                depfile = os.path.join(self.get_target_private_dir(target), depfilename)
+                args = [x.replace('@DEPFILE@', depfile)  for x in base_args]
             args = [x.replace("@INPUT@", infilename).replace('@OUTPUT@', sole_output)\
-                    for x in base_args]
+                    for x in args]
             args = self.replace_outputs(args, self.get_target_private_dir(target), outfilelist)
             # We have consumed output files, so drop them from the list of remaining outputs.
             if sole_output == '':
@@ -1365,7 +1389,9 @@ rule FORTRAN_DEP_HACK
             args = [x.replace("@SOURCE_DIR@", self.build_to_src).replace("@BUILD_DIR@", relout)
                     for x in args]
             cmdlist = exe_arr + self.replace_extra_args(args, genlist)
-            elem = NinjaBuildElement(self.all_outputs, outfiles, 'CUSTOM_COMMAND', infilename)
+            elem = NinjaBuildElement(self.all_outputs, outfiles, rulename, infilename)
+            if generator.depfile is not None:
+                elem.add_item('DEPFILE', depfile)
             if len(extra_dependencies) > 0:
                 elem.add_dep(extra_dependencies)
             elem.add_item('DESC', 'Generating $out')

mesonbuild/build.py

@@ -612,6 +612,7 @@ class Generator():
         if not isinstance(exe, (Executable, dependencies.ExternalProgram)):
             raise InvalidArguments('First generator argument must be an executable.')
         self.exe = exe
+        self.depfile = None
         self.process_kwargs(kwargs)
 
     def __repr__(self):
@@ -633,7 +634,6 @@ class Generator():
             if not isinstance(a, str):
                 raise InvalidArguments('A non-string object in "arguments" keyword argument.')
         self.arglist = args
-
         if 'output' not in kwargs:
             raise InvalidArguments('Generator must have "output" keyword argument.')
         outputs = kwargs['output']
@@ -651,12 +651,26 @@ class Generator():
                 if '@OUTPUT@' in o:
                     raise InvalidArguments('Tried to use @OUTPUT@ in a rule with more than one output.')
         self.outputs = outputs
+        if 'depfile' in kwargs:
+            depfile = kwargs['depfile']
+            if not isinstance(depfile, str):
+                raise InvalidArguments('Depfile must be a string.')
+            if os.path.split(depfile)[1] != depfile:
+                raise InvalidArguments('Depfile must be a plain filename without a subdirectory.')
+            self.depfile = depfile
 
     def get_base_outnames(self, inname):
         plainname = os.path.split(inname)[1]
         basename = plainname.split('.')[0]
         return [x.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname) for x in self.outputs]
 
+    def get_dep_outname(self, inname):
+        if self.depfile is None:
+            raise InvalidArguments('Tried to get dep name for rule that does not have dependency file defined.')
+        plainname = os.path.split(inname)[1]
+        basename = plainname.split('.')[0]
+        return self.depfile.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname)
+
     def get_arglist(self):
         return self.arglist
 
@@ -930,6 +944,7 @@ class CustomTarget:
                     'build_always' : True,
                     'depends' : True,
                     'depend_files' : True,
+                    'depfile' : True,
                     }
 
     def __init__(self, name, subdir, kwargs):
@@ -938,6 +953,7 @@ class CustomTarget:
         self.dependencies = []
         self.extra_depends = []
         self.depend_files = [] # Files that this target depends on but are not on the command line.
+        self.depfile = None
         self.process_kwargs(kwargs)
         self.extra_files = []
         self.install_rpath = ''
@@ -986,6 +1002,13 @@ class CustomTarget:
                 'Capturing can only output to a single file.')
         if 'command' not in kwargs:
             raise InvalidArguments('Missing keyword argument "command".')
+        if 'depfile' in kwargs:
+            depfile = kwargs['depfile']
+            if not isinstance(depfile, str):
+                raise InvalidArguments('Depfile must be a string.')
+            if os.path.split(depfile)[1] != depfile:
+                raise InvalidArguments('Depfile must be a plain filename without a subdirectory.')
+            self.depfile = depfile
         cmd = kwargs['command']
         if not(isinstance(cmd, list)):
             cmd = [cmd]

test cases/common/29 pipeline/meson.build

@@ -6,8 +6,9 @@ e1 = executable('srcgen', 'srcgen.c', native : true)
 
 # Generate a source file that needs to be included in the build.
 gen = generator(e1, \
- output  : '@BASENAME@.c', # Line continuation inside arguments should work without needing a "\".
+  depfile : '@BASENAME@.d',
- arguments : ['@INPUT@', '@OUTPUT@'])
+  output  : '@BASENAME@.c', # Line continuation inside arguments should work without needing a "\".
+  arguments : ['@INPUT@', '@OUTPUT@', '@DEPFILE@'])
 
 generated = gen.process(['input_src.dat'])
 

test cases/common/29 pipeline/srcgen.c

@@ -1,5 +1,6 @@
 #include<stdio.h>
 #include<assert.h>
+#include<string.h>
 
 #define ARRSIZE 80
 
@@ -7,17 +8,20 @@ int main(int argc, char **argv) {
     char arr[ARRSIZE];
     char *ofilename;
     char *ifilename;
+    char *dfilename;
     FILE *ifile;
     FILE *ofile;
+    FILE *depfile;
     size_t bytes;
+    int i;
 
-    if(argc != 3) {
+    if(argc != 4) {
-        fprintf(stderr, "%s <input file> <output file>\n", argv[0]);
+        fprintf(stderr, "%s <input file> <output file> <dependency file>\n", argv[0]);
         return 1;
     }
     ifilename = argv[1];
     ofilename = argv[2];
-    printf("%s\n", ifilename);
+    dfilename = argv[3];
     ifile = fopen(argv[1], "r");
     if(!ifile) {
         fprintf(stderr, "Could not open source file %s.\n", argv[1]);
@@ -34,7 +38,32 @@ int main(int argc, char **argv) {
     assert(bytes > 0);
     fwrite(arr, 1, bytes, ofile);
 
+    depfile = fopen(dfilename, "w");
+    if(!depfile) {
+        fprintf(stderr, "Could not open depfile %s\n", ofilename);
+        fclose(ifile);
+        fclose(ofile);
+        return 1;
+    }
+    for(i=0; i<strlen(ofilename); i++) {
+        if(ofilename[i] == ' ') {
+            fwrite("\\ ", 1, 2, depfile);
+        } else {
+            fwrite(&ofilename[i], 1, 1, depfile);
+        }
+    }
+    fwrite(": ", 1, 2, depfile);
+    for(i=0; i<strlen(ifilename); i++) {
+        if(ifilename[i] == ' ') {
+            fwrite("\\ ", 1, 2, depfile);
+        } else {
+            fwrite(&ifilename[i], 1, 1, depfile);
+        }
+    }
+    fwrite("\n", 1, 1, depfile);
+
     fclose(ifile);
     fclose(ofile);
+    fclose(depfile);
     return 0;
 }

test cases/common/56 custom target/depfile/dep.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+
+import sys, os
+from glob import glob
+
+_, srcdir, depfile, output = sys.argv
+
+depfiles = glob(os.path.join(srcdir, '*'))
+
+quoted_depfiles = [x.replace(' ', '\ ') for x in depfiles]
+
+open(output, 'w').write('I am the result of globbing.')
+open(depfile, 'w').write('%s: %s\n' % (output, ' '.join(quoted_depfiles)))

test cases/common/56 custom target/depfile/meson.build

@@ -0,0 +1,7 @@
+
+
+mytarget = custom_target('depfile',
+  output : 'dep.dat',
+  depfile : 'dep.dat.d',
+  command : [find_program('dep.py'), meson.current_source_dir(), '@DEPFILE@', '@OUTPUT@'],
+)

test cases/common/56 custom target/meson.build

@@ -13,3 +13,5 @@ command : [python, comp, '@INPUT@', '@OUTPUT@'],
 install : true,
 install_dir : 'subdir'
 )
+
+subdir('depfile')