Merge pull request #2764 from mesonbuild/generatorpath

Generator outputs can have path segments

docs/markdown/Reference-manual.md

@@ -1705,11 +1705,18 @@ This object is returned by [`generator()`](#generator) and contains a
 generator that is used to transform files from one type to another by
 an executable (e.g. `idl` files into source code and headers).
 
-* `process(list_of_files)` takes a list of files, causes them to be
-  processed and returns an object containing the result which can
+* `process(list_of_files, ...)` takes a list of files, causes them to
+  be processed and returns an object containing the result which can
   then, for example, be passed into a build target definition. The
   keyword argument `extra_args`, if specified, will be used to replace
-  an entry `@EXTRA_ARGS@` in the argument list.
+  an entry `@EXTRA_ARGS@` in the argument list. The keyword argument
+  `preserve_path_from`, if given, specifies that the output files need
+  to maintain their directory structure inside the target temporary
+  directory. The most common value for this is
+  `meson.current_source_dir()`. With this value when a file called
+  `subdir/one.input` is processed it generates a file `<target private
+  directory>/subdir/one.out` as opposed to `<target private
+  directory>/one.out`.
 
 ### `subproject` object
 

docs/markdown/snippets/gen-subdirs.md

@@ -0,0 +1,21 @@
+## Generator outputs can preserve directory structure
+
+Normally when generating files with a generator, Meson flattens the
+input files so they all go in the same directory. Some code
+generators, such as Protocol Buffers, require that the generated files
+have the same directory layout as the input files used to generate
+them. This can now be achieved like this:
+
+```meson
+g = generator(...) # Compiles protobuf sources
+generated = gen.process('com/mesonbuild/one.proto',
+  'com/mesonbuild/two.proto',
+  preserve_path_from : meson.current_source_dir())
+
+This would cause the following files to be generated inside the target
+private directory:
+
+    com/mesonbuild/one.pb.h
+    com/mesonbuild/one.pb.cc
+    com/mesonbuild/two.pb.h
+    com/mesonbuild/two.pb.cc

mesonbuild/backend/ninjabackend.py

@@ -1791,18 +1791,23 @@ rule FORTRAN_DEP_HACK
                 continue
             self.generate_genlist_for_target(genlist, target, outfile)
 
-    def replace_paths(self, target, args):
-        source_target_dir = self.get_target_source_dir(target)
+    def replace_paths(self, target, args, override_subdir=None):
+        if override_subdir:
+            source_target_dir = os.path.join(self.build_to_src, override_subdir)
+        else:
+            source_target_dir = self.get_target_source_dir(target)
         relout = self.get_target_private_dir(target)
         args = [x.replace("@SOURCE_DIR@", self.build_to_src).replace("@BUILD_DIR@", relout)
                 for x in args]
         args = [x.replace("@CURRENT_SOURCE_DIR@", source_target_dir) for x in args]
         args = [x.replace("@SOURCE_ROOT@", self.build_to_src).replace("@BUILD_ROOT@", '.')
                 for x in args]
+        args = [x.replace('\\', '/') for x in args]
         return args
 
     def generate_genlist_for_target(self, genlist, target, outfile):
         generator = genlist.get_generator()
+        subdir = genlist.subdir
         exe = generator.get_exe()
         exe_arr = self.exe_object_to_cmd_array(exe)
         infilelist = genlist.get_inputs()
@@ -1834,7 +1839,7 @@ rule FORTRAN_DEP_HACK
             if sole_output == '':
                 outfilelist = outfilelist[len(generator.outputs):]
             relout = self.get_target_private_dir(target)
-            args = self.replace_paths(target, args)
+            args = self.replace_paths(target, args, override_subdir=subdir)
             cmdlist = exe_arr + self.replace_extra_args(args, genlist)
             if generator.capture:
                 exe_data = self.serialize_executable(

mesonbuild/backend/vs2010backend.py

@@ -87,7 +87,6 @@ class Vs2010Backend(backends.Backend):
         custom_target_include_dirs = []
         custom_target_output_files = []
         target_private_dir = self.relpath(self.get_target_private_dir(target), self.get_target_dir(target))
-        source_target_dir = self.get_target_source_dir(target)
         down = self.target_to_build_root(target)
         for genlist in target.get_generated_sources():
             if isinstance(genlist, (build.CustomTarget, build.CustomTargetIndex)):
@@ -103,6 +102,7 @@ class Vs2010Backend(backends.Backend):
                 exe = generator.get_exe()
                 infilelist = genlist.get_inputs()
                 outfilelist = genlist.get_outputs()
+                source_dir = os.path.join(self.build_to_src, genlist.subdir)
                 exe_arr = self.exe_object_to_cmd_array(exe)
                 idgroup = ET.SubElement(parent_node, 'ItemGroup')
                 for i in range(len(infilelist)):
@@ -122,10 +122,11 @@ class Vs2010Backend(backends.Backend):
                     args = [x.replace("@SOURCE_DIR@", self.environment.get_source_dir())
                              .replace("@BUILD_DIR@", target_private_dir)
                             for x in args]
-                    args = [x.replace("@CURRENT_SOURCE_DIR@", source_target_dir) for x in args]
+                    args = [x.replace("@CURRENT_SOURCE_DIR@", source_dir) for x in args]
                     args = [x.replace("@SOURCE_ROOT@", self.environment.get_source_dir())
                              .replace("@BUILD_ROOT@", self.environment.get_build_dir())
                             for x in args]
+                    args = [x.replace('\\', '/') for x in args]
                     cmd = exe_arr + self.replace_extra_args(args, genlist)
                     if generator.capture:
                         exe_data = self.serialize_executable(

mesonbuild/build.py

@@ -14,7 +14,7 @@
 
 import copy, os, re
 from collections import OrderedDict
-import itertools
+import itertools, pathlib
 
 from . import environment
 from . import dependencies
@@ -1077,7 +1077,8 @@ class Generator:
     def get_base_outnames(self, inname):
         plainname = os.path.split(inname)[1]
         basename = os.path.splitext(plainname)[0]
-        return [x.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname) for x in self.outputs]
+        bases = [x.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname) for x in self.outputs]
+        return bases
 
     def get_dep_outname(self, inname):
         if self.depfile is None:
@@ -1091,32 +1092,54 @@ class Generator:
         basename = os.path.splitext(plainname)[0]
         return [x.replace('@BASENAME@', basename).replace('@PLAINNAME@', plainname) for x in self.arglist]
 
-    def process_files(self, name, files, state, extra_args=[]):
-        output = GeneratedList(self, extra_args=extra_args)
+    def is_parent_path(self, parent, trial):
+        relpath = pathlib.PurePath(trial).relative_to(parent)
+        return relpath.parts[0] != '..' # For subdirs we can only go "down".
+
+    def process_files(self, name, files, state, preserve_path_from=None, extra_args=[]):
+        output = GeneratedList(self, state.subdir, preserve_path_from, extra_args=extra_args)
         for f in files:
             if isinstance(f, str):
                 f = File.from_source_file(state.environment.source_dir, state.subdir, f)
             elif not isinstance(f, File):
                 raise InvalidArguments('{} arguments must be strings or files not {!r}.'.format(name, f))
-            output.add_file(f)
+            if preserve_path_from:
+                abs_f = f.absolute_path(state.environment.source_dir, state.environment.build_dir)
+                if not self.is_parent_path(preserve_path_from, abs_f):
+                    raise InvalidArguments('When using preserve_path_from, all input files must be in a subdirectory of the given dir.')
+            output.add_file(f, state)
         return output
 
 
 class GeneratedList:
-    def __init__(self, generator, extra_args=[]):
+    def __init__(self, generator, subdir, preserve_path_from=None, extra_args=[]):
         if hasattr(generator, 'held_object'):
             generator = generator.held_object
         self.generator = generator
         self.name = self.generator.exe
+        self.subdir = subdir
         self.infilelist = []
         self.outfilelist = []
         self.outmap = {}
         self.extra_depends = []
+        self.preserve_path_from = preserve_path_from
         self.extra_args = extra_args
 
-    def add_file(self, newfile):
+    def add_preserved_path_segment(self, infile, outfiles, state):
+        result = []
+        in_abs = infile.absolute_path(state.environment.source_dir, state.environment.build_dir)
+        assert(os.path.isabs(self.preserve_path_from))
+        rel = os.path.relpath(in_abs, self.preserve_path_from)
+        path_segment = os.path.split(rel)[0]
+        for of in outfiles:
+            result.append(os.path.join(path_segment, of))
+        return result
+
+    def add_file(self, newfile, state):
         self.infilelist.append(newfile)
         outfiles = self.generator.get_base_outnames(newfile.fname)
+        if self.preserve_path_from:
+            outfiles = self.add_preserved_path_segment(newfile, outfiles, state)
         self.outfilelist += outfiles
         self.outmap[newfile] = outfiles
 

mesonbuild/interpreter.py

@@ -375,7 +375,18 @@ class GeneratorHolder(InterpreterObject, ObjectHolder):
 
     def process_method(self, args, kwargs):
         extras = mesonlib.stringlistify(kwargs.get('extra_args', []))
-        gl = self.held_object.process_files('Generator', args, self.interpreter, extra_args=extras)
+        if 'preserve_path_from' in kwargs:
+            preserve_path_from = kwargs['preserve_path_from']
+            if not isinstance(preserve_path_from, str):
+                raise InvalidArguments('Preserve_path_from must be a string.')
+            preserve_path_from = os.path.normpath(preserve_path_from)
+            if not os.path.isabs(preserve_path_from):
+                # This is a bit of a hack. Fix properly before merging.
+                raise InvalidArguments('Preserve_path_from must be an absolute path for now. Sorry.')
+        else:
+            preserve_path_from = None
+        gl = self.held_object.process_files('Generator', args, self.interpreter,
+                                            preserve_path_from, extra_args=extras)
         return GeneratedListHolder(gl)
 
 
@@ -1372,7 +1383,7 @@ permitted_kwargs = {'add_global_arguments': {'language'},
                     'declare_dependency': {'include_directories', 'link_with', 'sources', 'dependencies', 'compile_args', 'link_args', 'version'},
                     'executable': exe_kwargs,
                     'find_program': {'required', 'native'},
-                    'generator': {'arguments', 'output', 'depfile', 'capture'},
+                    'generator': {'arguments', 'output', 'depfile', 'capture', 'preserve_path_from'},
                     'include_directories': {'is_system'},
                     'install_data': {'install_dir', 'install_mode', 'sources'},
                     'install_headers': {'install_dir', 'subdir'},

test cases/common/173 preserve gendir/base.inp

@@ -0,0 +1 @@
+base

test cases/common/173 preserve gendir/com/mesonbuild/subbie.inp

@@ -0,0 +1 @@
+subbie

test cases/common/173 preserve gendir/genprog.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import os, sys, argparse
+import pathlib
+
+h_templ = '''#pragma once
+
+int %s();
+'''
+
+c_templ = '''#include"%s.h"
+
+int %s() {
+    return 0;
+}
+'''
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--searchdir', required=True)
+parser.add_argument('--outdir', required=True)
+parser.add_argument('ifiles', nargs='+')
+
+options = parser.parse_args()
+
+searchdir = options.searchdir
+outdir = options.outdir
+ifiles = options.ifiles
+
+rel_ofiles = []
+
+for ifile in ifiles:
+    if not ifile.startswith(options.searchdir):
+        sys.exit('Input file %s does not start with search dir %s.' % (ifile, searchdir))
+    rel_ofile = ifile[len(searchdir):]
+    if rel_ofile[0] == '/' or rel_ofile[0] == '\\':
+        rel_ofile = rel_ofile[1:]
+    rel_ofiles.append(os.path.splitext(rel_ofile)[0])
+
+ofile_bases = [os.path.join(outdir, i) for i in rel_ofiles]
+
+for i, ifile_name in enumerate(ifiles):
+    proto_name = open(ifile_name).readline().strip()
+    h_out = ofile_bases[i] + '.h'
+    c_out = ofile_bases[i] + '.c'
+    os.makedirs(os.path.split(ofile_bases[i])[0], exist_ok=True)
+    open(h_out, 'w').write(h_templ % (proto_name))
+    open(c_out, 'w').write(c_templ % (proto_name, proto_name))

test cases/common/173 preserve gendir/meson.build

@@ -0,0 +1,13 @@
+project('preserve subdir', 'c')
+
+gprog = find_program('genprog.py')
+
+gen = generator(gprog, \
+  output    : ['@BASENAME@.c', '@BASENAME@.h'],
+  arguments : ['--searchdir=@CURRENT_SOURCE_DIR@', '--outdir=@BUILD_DIR@', '@INPUT@'])
+
+generated = gen.process('base.inp', 'com/mesonbuild/subbie.inp',
+  preserve_path_from : meson.current_source_dir())
+
+e = executable('testprog', 'testprog.c', generated)
+test('testprog', e)

test cases/common/173 preserve gendir/testprog.c

@@ -0,0 +1,6 @@
+#include"base.h"
+#include"com/mesonbuild/subbie.h"
+
+int main(int argc, char **argv) {
+    return base() + subbie();
+}

test cases/frameworks/5 protocol buffers/asubdir/defs.proto

@@ -1,3 +1,5 @@
+syntax = "proto3";
+
 message Dummy {
-  required string text = 1;
+  string text = 1;
 }

test cases/frameworks/5 protocol buffers/defs.proto

@@ -1,3 +1,5 @@
+syntax = "proto3";
+
 message Dummy {
-  required string text = 1;
+  string text = 1;
 }

test cases/frameworks/5 protocol buffers/meson.build

@@ -10,7 +10,7 @@ endif
 
 gen = generator(protoc, \
   output    : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'],
-  arguments : ['--proto_path=@SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'])
+  arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'])
 
 generated = gen.process('defs.proto')
 e = executable('prog', 'main.cpp', generated,
@@ -18,3 +18,5 @@ e = executable('prog', 'main.cpp', generated,
 test('prototest', e)
 
 subdir('asubdir')
+subdir('withpath')
+subdir('sidedir')

test cases/frameworks/5 protocol buffers/sidedir/meson.build

@@ -0,0 +1,7 @@
+# Generated source defined in one directory but
+# used in another.
+
+e = executable('sideprog', 'sideprog.cpp', generated,
+  override_options : ['unity=off'],
+  dependencies : dep)
+test('sideprog', e)

test cases/frameworks/5 protocol buffers/sidedir/sideprog.cpp

@@ -0,0 +1,16 @@
+#include"com/mesonbuild/simple.pb.h"
+#include"com/mesonbuild/subsite/complex.pb.h"
+
+#include<memory>
+
+int main(int argc, char **argv) {
+    GOOGLE_PROTOBUF_VERIFY_VERSION;
+    {
+        subdirectorial::SimpleMessage *s = new subdirectorial::SimpleMessage();
+        s->set_the_integer(3);
+        subdirectorial::ComplexMessage c;
+        c.set_allocated_sm(s);
+    }
+    google::protobuf::ShutdownProtobufLibrary();
+    return 0;
+}

test cases/frameworks/5 protocol buffers/withpath/com/mesonbuild/simple.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package subdirectorial;
+
+message SimpleMessage {
+  int32 the_integer = 1;
+}

test cases/frameworks/5 protocol buffers/withpath/com/mesonbuild/subsite/complex.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package subdirectorial;
+
+import "com/mesonbuild/simple.proto";
+
+message ComplexMessage {
+  string a_message = 1;
+  SimpleMessage sm = 2;
+}

test cases/frameworks/5 protocol buffers/withpath/meson.build

@@ -0,0 +1,13 @@
+# Testing protobuf files that are deeply hierarchical
+# and must preserve their path segments in output files
+# because protoc will always put it in there.
+
+generated = gen.process('com/mesonbuild/simple.proto',
+  'com/mesonbuild/subsite/complex.proto',
+  preserve_path_from : meson.current_source_dir(),
+  )
+
+e = executable('pathprog', 'pathprog.cpp', generated,
+  override_options : ['unity=off'],
+  dependencies : dep)
+test('pathprog', e)

test cases/frameworks/5 protocol buffers/withpath/pathprog.cpp

@@ -0,0 +1,16 @@
+#include"com/mesonbuild/simple.pb.h"
+#include"com/mesonbuild/subsite/complex.pb.h"
+
+#include<memory>
+
+int main(int argc, char **argv) {
+    GOOGLE_PROTOBUF_VERIFY_VERSION;
+    {
+        subdirectorial::SimpleMessage *s = new subdirectorial::SimpleMessage();
+        s->set_the_integer(3);
+        subdirectorial::ComplexMessage c;
+        c.set_allocated_sm(s);
+    }
+    google::protobuf::ShutdownProtobufLibrary();
+    return 0;
+}