Support all kinds of generated vala and vapi sources

This is the first step in making Vala support have feature-parity with
C/C++ support. Vala and Vapi sources generated with Generators and
CustomTargets are no longer ignored. Dependencies are setup properly and
they are added to the commandline.

mesonbuild/backend/ninjabackend.py

@@ -264,7 +264,6 @@ int dummy;
         if isinstance(target, build.RunTarget):
             self.generate_run_target(target, outfile)
         name = target.get_id()
-        vala_gen_sources = []
         if name in self.processed_targets:
             return
         self.processed_targets[name] = True
@@ -285,12 +284,20 @@ int dummy;
             self.generate_swift_target(target, outfile)
             return
 
-        # Pre-existing target C/C++ sources to be built
-        target_sources = []
-        # GeneratedList and CustomTarget sources to be built
-        generated_sources = []
+        # Pre-existing target C/C++ sources to be built; dict of full path to
+        # source relative to build root and the original File object.
+        target_sources = {}
+        # GeneratedList and CustomTarget sources to be built; dict of the full
+        # path to source relative to build root and the generating target/list
+        generated_sources = {}
+        # Array of sources generated by valac that have to be compiled
+        vala_generated_sources = []
         if 'vala' in target.compilers:
-            vala_gen_sources = self.generate_vala_compile(target, outfile)
+            # Sources consumed by valac are filtered out. These only contain
+            # C/C++ sources, objects, generated libs, and unknown sources now.
+            target_sources, generated_sources, \
+            vala_generated_sources = self.generate_vala_compile(target, outfile)
+        else:
             target_sources = self.get_target_sources(target)
             generated_sources = self.get_target_generated_sources(target)
         self.scan_fortran_module_outputs(target)
@@ -347,8 +354,8 @@ int dummy;
                                                          header_deps=header_deps))
         # Generate compilation targets for C sources generated from Vala
         # sources. This can be extended to other $LANG->C compilers later if
-        # necessary.
-        for src in vala_gen_sources:
+        # necessary. This needs to be separate for at least Vala
+        for src in vala_generated_sources:
             src_list.append(src)
             if is_unity:
                 unity_src.append(os.path.join(self.environment.get_build_dir(), src))
@@ -368,8 +375,6 @@ int dummy;
                     obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))
         # Generate compile targets for all the pre-existing sources for this target
         for f, src in target_sources.items():
-            if src.endswith('.vala'):
-                continue
             if not self.environment.is_header(src):
                 src_list.append(src)
                 if is_unity:
@@ -871,17 +876,13 @@ int dummy;
         outfile.write(description)
         outfile.write('\n')
 
-    def split_vala_sources(self, sources):
-        other_src = []
-        vapi_src = []
-        for s in sources:
-            if s.endswith('.vapi'):
-                vapi_src.append(s)
-            else:
-                other_src.append(s)
-        return (other_src, vapi_src)
-
     def determine_dep_vapis(self, target):
+        """
+        Peek into the sources of BuildTargets we're linking with, and if any of
+        them was built with Vala, assume that it also generated a .vapi file of
+        the same name as the BuildTarget and return the path to it relative to
+        the build directory.
+        """
         result = []
         for dep in target.link_targets:
             for i in dep.sources:
@@ -894,54 +895,116 @@ int dummy;
                     break
         return result
 
+    def split_vala_sources(self, t):
+        """
+        Splits the target's sources into .vala, .vapi, and other sources.
+        Handles both pre-existing and generated sources.
+
+        Returns a tuple (vala, vapi, others) each of which is a dictionary with
+        the keys being the path to the file (relative to the build directory)
+        and the value being the object that generated or represents the file.
+        """
+        vala = {}
+        vapi = {}
+        others = {}
+        othersgen = {}
+        # Split pre-existing sources
+        for s in t.get_sources():
+            # BuildTarget sources are always mesonlib.File files which are
+            # either in the source root, or generated with configure_file and
+            # in the build root
+            if not isinstance(s, File):
+                msg = 'All sources in target {!r} must be of type ' \
+                      'mesonlib.File, not {!r}'.format(t, s)
+                raise InvalidArguments(msg)
+            f = s.rel_to_builddir(self.build_to_src)
+            if s.endswith('.vala'):
+                srctype = vala
+            elif s.endswith('.vapi'):
+                srctype = vapi
+            else:
+                srctype = others
+            srctype[f] = s
+        # Split generated sources
+        for gensrc in t.get_generated_sources():
+            for s in gensrc.get_outputs():
+                f = self.get_target_generated_dir(t, gensrc, s)
+                if s.endswith('.vala'):
+                    srctype = vala
+                elif s.endswith('.vapi'):
+                    srctype = vapi
+                # Generated non-Vala (C/C++) sources. Won't be used for
+                # generating the Vala compile rule below.
+                else:
+                    srctype = othersgen
+                # Duplicate outputs are disastrous
+                if f in srctype:
+                    msg = 'Duplicate output {0!r} from {1!r} {2!r}; ' \
+                          'conflicts with {0!r} from {4!r} {3!r}' \
+                          ''.format(f, type(gensrc).__name__, gensrc.name,
+                                    srctype[f].name, type(srctype[f]).__name__)
+                    raise InvalidArguments(msg)
+                # Store 'somefile.vala': GeneratedList (or CustomTarget)
+                srctype[f] = gensrc
+        return (vala, vapi, (others, othersgen))
+
     def generate_vala_compile(self, target, outfile):
         """Vala is compiled into C. Set up all necessary build steps here."""
-        valac = target.compilers['vala']
-        (other_src, vapi_src) = self.split_vala_sources(target.get_sources())
-        vapi_src = [x.rel_to_builddir(self.build_to_src) for x in vapi_src]
+        (vala_src, vapi_src, other_src) = self.split_vala_sources(target)
         extra_dep_files = []
-        if len(other_src) == 0:
+        if len(vala_src) == 0:
             raise InvalidArguments('Vala library has no Vala source files.')
-        namebase = target.name
-        base_h = namebase + '.h'
-        base_vapi = namebase + '.vapi'
-        hname = os.path.normpath(os.path.join(self.get_target_dir(target), base_h))
-        vapiname = os.path.normpath(os.path.join(self.get_target_dir(target), base_vapi))
-
-        generated_c_files = []
-        outputs = [vapiname]
-        args = []
-        args += self.build.get_global_args(valac)
-        args += valac.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype'))
-        args += ['-d', self.get_target_private_dir(target)]
-        args += ['-C']#, '-o', cname]
-        if not isinstance(target, build.Executable):
-            outputs.append(hname)
-            args += ['-H', hname]
-            args += ['--library=' + target.name]
-        args += ['--vapi=' + os.path.join('..', base_vapi)]
-        vala_src = []
-        for s in other_src:
-            if not s.endswith('.vala'):
-                continue
-            vala_file = s.rel_to_builddir(self.build_to_src)
-            vala_src.append(vala_file)
+
+        valac = target.compilers['vala']
+        c_out_dir = self.get_target_private_dir(target)
+        # C files generated by valac
+        vala_c_src = []
+        # Files generated by valac
+        valac_outputs = []
+        # All sources that are passed to valac on the commandline
+        all_files = list(vapi_src.keys())
+        for (vala_file, gensrc) in vala_src.items():
+            all_files.append(vala_file)
             # Figure out where the Vala compiler will write the compiled C file
-            dirname, basename = os.path.split(vala_file)
             # If the Vala file is in a subdir of the build dir (in our case
             # because it was generated/built by something else), the subdir path
             # components will be preserved in the output path. But if the Vala
             # file is outside the build directory, the path components will be
             # stripped and just the basename will be used.
-            c_file = os.path.splitext(basename)[0] + '.c'
-            if s.is_built:
-                c_file = os.path.join(dirname, c_file)
-            full_c = os.path.join(self.get_target_private_dir(target), c_file)
-            generated_c_files.append(full_c)
-            outputs.append(full_c)
+            if isinstance(gensrc, (build.CustomTarget, build.GeneratedList)) or gensrc.is_built:
+                vala_c_file = os.path.splitext(vala_file)[0] + '.c'
+            else:
+                vala_c_file = os.path.splitext(os.path.basename(vala_file))[0] + '.c'
+            # All this will be placed inside the c_out_dir
+            vala_c_file = os.path.join(c_out_dir, vala_c_file)
+            vala_c_src.append(vala_c_file)
+            valac_outputs.append(vala_c_file)
+
+        args = []
+        args += self.build.get_global_args(valac)
+        args += valac.get_buildtype_args(self.environment.coredata.get_builtin_option('buildtype'))
+        # Tell Valac to output everything in our private directory. Sadly this
+        # means it will also preserve the directory components of Vala sources
+        # found inside the build tree (generated sources).
+        args += ['-d', c_out_dir]
+        args += ['-C']
+        if not isinstance(target, build.Executable):
+            # Library name
+            args += ['--library=' + target.name]
+            # Outputted header
+            hname = os.path.join(self.get_target_dir(target), target.name + '.h')
+            args += ['-H', hname]
+            valac_outputs.append(hname)
+            # Outputted vapi file
+            base_vapi = target.name + '.vapi'
+            vapiname = os.path.join(self.get_target_dir(target), base_vapi)
+            # Force valac to write the vapi file in the target build dir.
+            # Without this, it will write it inside c_out_dir
+            args += ['--vapi=../' + base_vapi]
+            valac_outputs.append(vapiname)
         if self.environment.coredata.get_builtin_option('werror'):
             args += valac.get_werror_args()
-        for d in target.external_deps:
+        for d in target.get_external_deps():
             if isinstance(d, dependencies.PkgConfigDependency):
                 if d.name == 'glib-2.0' and d.version_requirement is not None \
                    and d.version_requirement.startswith(('>=', '==')):
@@ -959,14 +1022,13 @@ int dummy;
         dependency_vapis = self.determine_dep_vapis(target)
         extra_dep_files += dependency_vapis
         args += extra_args
-        args += dependency_vapis
-        element = NinjaBuildElement(self.all_outputs, outputs,
+        element = NinjaBuildElement(self.all_outputs, valac_outputs,
                                     valac.get_language() + '_COMPILER',
-                                    vala_src + vapi_src)
+                                    all_files + dependency_vapis)
         element.add_item('ARGS', args)
         element.add_dep(extra_dep_files)
         element.write(outfile)
-        return generated_c_files
+        return other_src[0], other_src[1], vala_c_src
 
     def generate_rust_target(self, target, outfile):
         rustc = target.compilers['rust']
@@ -1664,7 +1726,7 @@ rule FORTRAN_DEP_HACK
             if isinstance(src, File):
                 rel_src = src.rel_to_builddir(self.build_to_src)
             else:
-                raise build.InvalidArguments('Invalid source type.')
+                raise InvalidArguments('Invalid source type: {!r}'.format(src))
             abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
         if isinstance(src, (RawFilename, File)):
             src_filename = src.fname
@@ -1791,7 +1853,10 @@ rule FORTRAN_DEP_HACK
             if len(pch) == 0:
                 continue
             if '/' not in pch[0] or '/' not in pch[-1]:
-                raise build.InvalidArguments('Precompiled header of "%s" must not be in the same directory as source, please put it in a subdirectory.' % target.get_basename())
+                msg = 'Precompiled header of {!r} must not be in the same ' \
+                      'directory as source, please put it in a subdirectory.' \
+                      ''.format(target.get_basename())
+                raise InvalidArguments(msg)
             compiler = self.get_compiler_for_lang(lang)
             if compiler.id == 'msvc':
                 src = os.path.join(self.build_to_src, target.get_source_subdir(), pch[-1])

mesonbuild/build.py

@@ -776,6 +776,7 @@ class GeneratedList():
         if hasattr(generator, 'held_object'):
             generator = generator.held_object
         self.generator = generator
+        self.name = self.generator.exe
         self.infilelist = []
         self.outfilelist = []
         self.outmap = {}

test cases/vala/8 generated source/src/meson.build

@@ -1,5 +0,0 @@
-config = configure_file(input: 'config.vala.in',
-  output: 'config.vala',
-  configuration: cd)
-
-src = files('test.vala')

test cases/vala/8 generated source/src/test.vala

@@ -1,3 +0,0 @@
-void main() {
-    print (Config.x);
-}

test cases/vala/8 generated sources/src/copy_file.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import shutil
+
+shutil.copyfile(sys.argv[1], sys.argv[2])

test cases/vala/8 generated sources/src/meson.build

@@ -0,0 +1,17 @@
+config = configure_file(input: 'config.vala.in',
+  output: 'config.vala',
+  configuration: cd)
+
+print = find_program('write_wrapper.py')
+wrapper = custom_target('wrapper',
+  output : 'print_wrapper.vala',
+  command : [print, '@OUTPUT@'])
+
+copy = find_program('copy_file.py')
+gen = generator(copy,
+  output : '@BASENAME@.vala',
+  arguments : ['@INPUT@', '@OUTPUT@'])
+
+returncode = gen.process('returncode.in')
+
+src = files('test.vala')

test cases/vala/8 generated sources/src/returncode.in

@@ -0,0 +1,3 @@
+int return_code() {
+    return 0;
+}

test cases/vala/8 generated sources/src/test.vala

@@ -0,0 +1,4 @@
+int main() {
+    print_wrapper (Config.x);
+    return return_code ();
+}

test cases/vala/8 generated sources/src/write_wrapper.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import sys
+
+contents = '''
+void print_wrapper(string arg) {
+    print (arg);
+}
+'''
+
+with open(sys.argv[1], 'w') as f:
+    f.write(contents)

test cases/vala/8 generated sources/tools/meson.build

@@ -1,3 +1,3 @@
-executable('generatedtest', [src, config],
+executable('generatedtest', [src, config, returncode, wrapper],
   install : true,
   dependencies: [dependency('glib-2.0'), dependency('gobject-2.0')])