Merge pull request #938 from centricular/fix-unity-builds

Several commits that fix unity builds

mesonbuild/backend/backends.py

@@ -19,7 +19,7 @@ from .. import mesonlib
 from .. import compilers
 import json
 import subprocess
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, get_compiler_for_source, classify_unity_sources
 
 class InstallData():
     def __init__(self, source_dir, build_dir, prefix):
@@ -78,21 +78,6 @@ class Backend():
             priv_dirname = self.get_target_private_dir_abs(t)
             os.makedirs(priv_dirname, exist_ok=True)
 
-    def get_compiler_for_lang(self, lang):
-        for i in self.build.compilers:
-            if i.language == lang:
-                return i
-        raise RuntimeError('No compiler for language ' + lang)
-
-    def get_compiler_for_source(self, src, is_cross):
-        comp = self.build.cross_compilers if is_cross else self.build.compilers
-        for i in comp:
-            if i.can_compile(src):
-                return i
-        if isinstance(src, mesonlib.File):
-            src = src.fname
-        raise RuntimeError('No specified compiler can handle file ' + src)
-
     def get_target_filename(self, t):
         if isinstance(t, build.CustomTarget):
             if len(t.get_outputs()) != 1:
@@ -153,14 +138,17 @@ class Backend():
         # target that the GeneratedList is used in
         return os.path.join(self.get_target_private_dir(target), src)
 
+    def get_unity_source_filename(self, target, suffix):
+        return target.name + '-unity.' + suffix
+
     def generate_unity_files(self, target, unity_src):
-        langlist = {}
         abs_files = []
         result = []
+        compsrcs = classify_unity_sources(target.compilers.values(), unity_src)
 
-        def init_language_file(language, suffix):
+        def init_language_file(suffix):
             outfilename = os.path.join(self.get_target_private_dir_abs(target),
-                                       target.name + '-unity' + suffix)
+                                       self.get_unity_source_filename(target, suffix))
             outfileabs = os.path.join(self.environment.get_build_dir(),
                                       outfilename)
             outfileabs_tmp = outfileabs + '.tmp'
@@ -171,20 +159,12 @@ class Backend():
             result.append(outfilename)
             return open(outfileabs_tmp, 'w')
 
-        try:
-            for src in unity_src:
-                comp = self.get_compiler_for_source(src, target.is_cross)
-                language = comp.get_language()
-                try:
-                    ofile = langlist[language]
-                except KeyError:
-                    suffix = '.' + comp.get_default_suffix()
-                    ofile = langlist[language] = init_language_file(language,
-                                                                    suffix)
-                ofile.write('#include<%s>\n' % src)
-        finally:
-            for x in langlist.values():
-                x.close()
+        # For each language, generate a unity source file and return the list
+        for comp, srcs in compsrcs.items():
+            lang = comp.get_language()
+            with init_language_file(comp.get_default_suffix()) as ofile:
+                for src in srcs:
+                    ofile.write('#include<%s>\n' % src)
         [mesonlib.replace_if_different(x, x + '.tmp') for x in abs_files]
         return result
 
@@ -278,24 +258,37 @@ class Backend():
             for s in src:
                 if c.can_compile(s):
                     return c
-        raise RuntimeError('Unreachable code')
+        raise AssertionError("BUG: Couldn't determine linker for sources {!r}".format(src))
 
     def object_filename_from_source(self, target, source):
-        return source.fname.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
+        if isinstance(source, mesonlib.File):
+            source = source.fname
+        return source.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
 
-    def determine_ext_objs(self, extobj, proj_dir_to_build_root=''):
+    def determine_ext_objs(self, extobj, proj_dir_to_build_root):
         result = []
         targetdir = self.get_target_private_dir(extobj.target)
+        # With unity builds, there's just one object that contains all the
+        # sources, so if we want all the objects, just return that.
+        if self.environment.coredata.get_builtin_option('unity'):
+            if not extobj.unity_compatible:
+                # This should never happen
+                msg = 'BUG: Meson must not allow extracting single objects ' \
+                      'in Unity builds'
+                raise AssertionError(msg)
+            comp = get_compiler_for_source(extobj.target.compilers.values(),
+                                           extobj.srclist[0])
+            # The unity object name uses the full absolute path of the source file
+            osrc = os.path.join(self.get_target_private_dir_abs(extobj.target),
+                                self.get_unity_source_filename(extobj.target,
+                                                               comp.get_default_suffix()))
+            objname = self.object_filename_from_source(extobj.target, osrc)
+            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
+            return [objpath]
         for osrc in extobj.srclist:
-            # If extracting in a subproject, the subproject
-            # name gets duplicated in the file name.
-            pathsegs = osrc.subdir.split(os.sep)
-            if pathsegs[0] == 'subprojects':
-                pathsegs = pathsegs[2:]
-            fixedpath = os.sep.join(pathsegs)
-            objname = os.path.join(proj_dir_to_build_root, targetdir,
-                                   self.object_filename_from_source(extobj.target, osrc))
-            result.append(objname)
+            objname = self.object_filename_from_source(extobj.target, osrc)
+            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
+            result.append(objpath)
         return result
 
     def get_pch_include_args(self, compiler, target):

mesonbuild/backend/ninjabackend.py

@@ -18,7 +18,7 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..mesonlib import File, MesonException
+from ..mesonlib import File, MesonException, get_compiler_for_source
 from .backends import InstallData
 from ..build import InvalidArguments
 import os, sys, pickle, re
@@ -258,6 +258,20 @@ int dummy;
             srcs[f] = s
         return srcs
 
+    # Languages that can mix with C or C++ but don't support unity builds yet
+    # because the syntax we use for unity builds is specific to C/++/ObjC/++.
+    langs_cant_unity = ('d', 'fortran')
+    def get_target_source_can_unity(self, target, source):
+        if isinstance(source, File):
+            source = source.fname
+        suffix = os.path.splitext(source)[1][1:]
+        for lang in self.langs_cant_unity:
+            if not lang in target.compilers:
+                continue
+            if suffix in target.compilers[lang].file_suffixes:
+                return False
+        return True
+
     def generate_target(self, target, outfile):
         if isinstance(target, build.CustomTarget):
             self.generate_custom_target(target, outfile)
@@ -319,6 +333,18 @@ int dummy;
         header_deps += self.get_generated_headers(target)
         src_list = []
 
+        if is_unity:
+            # Warn about incompatible sources if a unity build is enabled
+            langs = set(target.compilers.keys())
+            langs_cant = langs.intersection(self.langs_cant_unity)
+            if langs_cant:
+                langs_are = langs = ', '.join(langs_cant).upper()
+                langs_are += ' are' if len(langs_cant) > 1 else ' is'
+                msg = '{} not supported in Unity builds yet, so {} ' \
+                      'sources in the {!r} target will be compiled normally' \
+                      ''.format(langs_are, langs, target.name)
+                mlog.log(mlog.red('FIXME'), msg)
+
         # Get a list of all generated *sources* (sources files, headers,
         # objects, etc). Needed to determine the linker.
         generated_output_sources = [] 
@@ -329,13 +355,14 @@ int dummy;
         generated_source_files = []
         for rel_src, gensrc in generated_sources.items():
             generated_output_sources.append(rel_src)
+            raw_src = RawFilename(rel_src)
             if self.environment.is_source(rel_src) and not self.environment.is_header(rel_src):
-                if is_unity:
-                    unity_deps.append(rel_src)
+                if is_unity and self.get_target_source_can_unity(target, rel_src):
+                    unity_deps.append(raw_src)
                     abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
                     unity_src.append(abs_src)
                 else:
-                    generated_source_files.append(RawFilename(rel_src))
+                    generated_source_files.append(raw_src)
             elif self.environment.is_object(rel_src):
                 obj_list.append(rel_src)
             elif self.environment.is_library(rel_src):
@@ -344,7 +371,7 @@ int dummy;
                 # Assume anything not specifically a source file is a header. This is because
                 # people generate files with weird suffixes (.inc, .fh) that they then include
                 # in their source files.
-                header_deps.append(RawFilename(rel_src))
+                header_deps.append(raw_src)
         # These are the generated source files that need to be built for use by
         # this target. We create the Ninja build file elements for this here
         # because we need `header_deps` to be fully generated in the above loop.
@@ -352,14 +379,17 @@ int dummy;
             src_list.append(src)
             obj_list.append(self.generate_single_compile(target, outfile, src, True,
                                                          header_deps=header_deps))
+
         # Generate compilation targets for C sources generated from Vala
         # sources. This can be extended to other $LANG->C compilers later if
         # necessary. This needs to be separate for at least Vala
+        vala_generated_source_files = []
         for src in vala_generated_sources:
+            raw_src = RawFilename(src)
             src_list.append(src)
             if is_unity:
                 unity_src.append(os.path.join(self.environment.get_build_dir(), src))
-                header_deps.append(src)
+                header_deps.append(raw_src)
             else:
                 # Generated targets are ordered deps because the must exist
                 # before the sources compiling them are used. After the first
@@ -367,17 +397,22 @@ int dummy;
                 # This should work in all cases. If it does not, then just
                 # move them from orderdeps to proper deps.
                 if self.environment.is_header(src):
-                    header_deps.append(src)
+                    header_deps.append(raw_src)
                 else:
-                    # Passing 'vala' here signifies that we want the compile
-                    # arguments to be specialized for C code generated by
-                    # valac. For instance, no warnings should be emitted.
-                    obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))
+                    # We gather all these and generate compile rules below
+                    # after `header_deps` (above) is fully generated
+                    vala_generated_source_files.append(raw_src)
+        for src in vala_generated_source_files:
+            # Passing 'vala' here signifies that we want the compile
+            # arguments to be specialized for C code generated by
+            # valac. For instance, no warnings should be emitted.
+            obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))
+
         # Generate compile targets for all the pre-existing sources for this target
         for f, src in target_sources.items():
             if not self.environment.is_header(src):
                 src_list.append(src)
-                if is_unity:
+                if is_unity and self.get_target_source_can_unity(target, src):
                     abs_src = os.path.join(self.environment.get_build_dir(),
                                            src.rel_to_builddir(self.build_to_src))
                     unity_src.append(abs_src)
@@ -386,7 +421,7 @@ int dummy;
         obj_list += self.flatten_object_list(target)
         if is_unity:
             for src in self.generate_unity_files(target, unity_src):
-                obj_list.append(self.generate_single_compile(target, outfile, src, True, unity_deps + header_deps))
+                obj_list.append(self.generate_single_compile(target, outfile, RawFilename(src), True, unity_deps + header_deps))
         linker = self.determine_linker(target, src_list + generated_output_sources)
         elem = self.generate_link(target, outfile, outname, obj_list, linker, pch_objects)
         self.generate_shlib_aliases(target, self.get_target_dir(target))
@@ -1668,14 +1703,14 @@ rule FORTRAN_DEP_HACK
 
     def generate_single_compile(self, target, outfile, src, is_generated=False, header_deps=[], order_deps=[]):
         """
-        Compiles only C/C++ and ObjC/ObjC++ sources
+        Compiles C/C++, ObjC/ObjC++, and D sources
         """
-        if(isinstance(src, str) and src.endswith('.h')):
-            raise RuntimeError('Fug')
+        if isinstance(src, str) and src.endswith('.h'):
+            raise AssertionError('BUG: sources should not contain headers')
         if isinstance(src, RawFilename) and src.fname.endswith('.h'):
-            raise RuntimeError('Fug')
+            raise AssertionError('BUG: sources should not contain headers')
         extra_orderdeps = []
-        compiler = self.get_compiler_for_source(src, target.is_cross)
+        compiler = get_compiler_for_source(target.compilers.values(), src)
         commands = []
         # The first thing is implicit include directories: source, build and private.
         commands += compiler.get_include_args(self.get_target_private_dir(target), False)
@@ -1716,12 +1751,12 @@ rule FORTRAN_DEP_HACK
                 break
         if isinstance(src, RawFilename):
             rel_src = src.fname
-        elif is_generated:
-            if self.has_dir_part(src):
-                rel_src = src
+            if os.path.isabs(src.fname):
+                abs_src = src.fname
             else:
-                rel_src = os.path.join(self.get_target_private_dir(target), src)
-                abs_src = os.path.join(self.environment.get_source_dir(), rel_src)
+                abs_src = os.path.join(self.environment.get_build_dir(), src.fname)
+        elif is_generated:
+            raise AssertionError('BUG: broken generated source file handling for {!r}'.format(src))
         else:
             if isinstance(src, File):
                 rel_src = src.rel_to_builddir(self.build_to_src)
@@ -1805,6 +1840,7 @@ rule FORTRAN_DEP_HACK
         return rel_obj
 
     def has_dir_part(self, fname):
+        # FIXME FIXME: The usage of this is a terrible and unreliable hack
         return '/' in fname or '\\' in fname
 
     # Fortran is a bit weird (again). When you link against a library, just compiling a source file
@@ -1857,7 +1893,7 @@ rule FORTRAN_DEP_HACK
                       'directory as source, please put it in a subdirectory.' \
                       ''.format(target.get_basename())
                 raise InvalidArguments(msg)
-            compiler = self.get_compiler_for_lang(lang)
+            compiler = target.compilers[lang]
             if compiler.id == 'msvc':
                 src = os.path.join(self.build_to_src, target.get_source_subdir(), pch[-1])
                 (commands, dep, dst, objs) = self.generate_msvc_pch_command(target, compiler, pch)

mesonbuild/build.py

@@ -17,7 +17,7 @@ from . import environment
 from . import dependencies
 from . import mlog
 import copy, os, re
-from .mesonlib import File, flatten, MesonException, stringlistify
+from .mesonlib import File, flatten, MesonException, stringlistify, classify_unity_sources
 from .environment import for_windows, for_darwin
 
 known_basic_kwargs = {'install' : True,
@@ -176,9 +176,42 @@ class IncludeDirs():
         return self.extra_build_dirs
 
 class ExtractedObjects():
+    '''
+    Holds a list of sources for which the objects must be extracted
+    '''
     def __init__(self, target, srclist):
         self.target = target
         self.srclist = srclist
+        self.check_unity_compatible()
+
+    def check_unity_compatible(self):
+        # Figure out if the extracted object list is compatible with a Unity
+        # build. When we're doing a Unified build, we go through the sources,
+        # and create a single source file from each subset of the sources that
+        # can be compiled with a specific compiler. Then we create one object
+        # from each unified source file.
+        # If the list of sources for which we want objects is the same as the
+        # list of sources that go into each unified build, we're good.
+        self.unity_compatible = False
+        srclist_set = set(self.srclist)
+        # Objects for all the sources are required, so we're compatible
+        if srclist_set == set(self.target.sources):
+            self.unity_compatible = True
+            return
+        # Check if the srclist is a subset (of the target's sources) that is
+        # going to form a unified source file and a single object
+        compsrcs = classify_unity_sources(self.target.compilers.values(),
+                                          self.target.sources)
+        for srcs in compsrcs.values():
+            if srclist_set == set(srcs):
+                self.unity_compatible = True
+                return
+        msg = 'Single object files can not be extracted in Unity builds. ' \
+              'You can only extract all the object files at once.'
+        raise MesonException(msg)
+
+    def get_want_all_objects(self):
+        return self.want_all_objects
 
 class EnvironmentVariables():
     def __init__(self):
@@ -312,6 +345,13 @@ class BuildTarget():
                 msg = 'Bad source of type {!r} in target {!r}.'.format(type(s).__name__, self.name)
                 raise InvalidArguments(msg)
 
+    @staticmethod
+    def can_compile_sources(compiler, sources):
+        for s in sources:
+            if compiler.can_compile(s):
+                return True
+        return False
+
     @staticmethod
     def can_compile_remove_sources(compiler, sources):
         removed = False
@@ -322,16 +362,23 @@ class BuildTarget():
         return removed
 
     def process_compilers(self):
-        if len(self.sources) == 0:
+        if len(self.sources) + len(self.generated) == 0:
             return
         sources = list(self.sources)
+        for gensrc in self.generated:
+            sources += gensrc.get_outputs()
+        # Populate list of compilers
         if self.is_cross:
             compilers = self.environment.coredata.cross_compilers
         else:
             compilers = self.environment.coredata.compilers
         for lang, compiler in compilers.items():
-            if self.can_compile_remove_sources(compiler, sources):
+            if self.can_compile_sources(compiler, sources):
                 self.compilers[lang] = compiler
+        # If all our sources are Vala, our target also needs the C compiler but
+        # it won't get added above.
+        if 'vala' in self.compilers and 'c' not in self.compilers:
+            self.compilers['c'] = compilers['c']
 
     def validate_sources(self):
         if len(self.sources) == 0:
@@ -383,18 +430,15 @@ class BuildTarget():
         if 'link_with' in self.kwargs:
             self.kwargs['link_with'] = self.unpack_holder(self.kwargs['link_with'])
 
-    def extract_objects(self, srcargs):
+    def extract_objects(self, srclist):
         obj_src = []
-        for srclist in srcargs:
-            if not isinstance(srclist, list):
-                srclist = [srclist]
-            for src in srclist:
-                if not isinstance(src, str):
-                    raise MesonException('Extraction arguments must be strings.')
-                src = File(False, self.subdir, src)
-                if src not in self.sources:
-                    raise MesonException('Tried to extract unknown source %s.' % src)
-                obj_src.append(src)
+        for src in srclist:
+            if not isinstance(src, str):
+                raise MesonException('Object extraction arguments must be strings.')
+            src = File(False, self.subdir, src)
+            if src not in self.sources:
+                raise MesonException('Tried to extract unknown source %s.' % src)
+            obj_src.append(src)
         return ExtractedObjects(self, obj_src)
 
     def extract_all_objects(self):

mesonbuild/compilers.py

@@ -342,6 +342,9 @@ class Compiler():
     def get_language(self):
         return self.language
 
+    def get_default_suffix(self):
+        return self.default_suffix
+
     def get_exelist(self):
         return self.exelist[:]
 
@@ -497,9 +500,6 @@ class CCompiler(Compiler):
     def get_depfile_suffix(self):
         return 'd'
 
-    def get_default_suffix(self):
-        return self.default_suffix
-
     def get_exelist(self):
         return self.exelist[:]
 
@@ -1106,9 +1106,6 @@ class MonoCompiler(Compiler):
     def get_dependency_gen_args(self, outtarget, outfile):
         return []
 
-    def get_default_suffix(self):
-        return self.default_suffix
-
     def get_linker_exelist(self):
         return self.exelist[:]
 
@@ -1193,9 +1190,6 @@ class JavaCompiler(Compiler):
     def get_dependency_gen_args(self, outtarget, outfile):
         return []
 
-    def get_default_suffix(self):
-        return self.default_suffix
-
     def get_linker_exelist(self):
         return self.exelist[:]
 
@@ -1842,7 +1836,6 @@ class VisualStudioCPPCompiler(VisualStudioCCompiler):
     def __init__(self, exelist, version, is_cross, exe_wrap):
         self.language = 'cpp'
         VisualStudioCCompiler.__init__(self, exelist, version, is_cross, exe_wrap)
-        self.default_suffix = 'cpp'
         self.base_options = ['b_pch'] # FIXME add lto, pgo and the like
 
     def get_options(self):

mesonbuild/interpreter.py

@@ -2571,8 +2571,6 @@ requirements use the version keyword argument instead.''')
         else:
             obj = self.evaluate_statement(invokable)
         method_name = node.name
-        if method_name == 'extract_objects' and self.environment.coredata.get_builtin_option('unity'):
-            raise InterpreterException('Single object files can not be extracted in Unity builds.')
         args = node.args
         if isinstance(obj, mparser.StringNode):
             obj = obj.get_value()

mesonbuild/mesonlib.py

@@ -70,6 +70,22 @@ class File:
     def __hash__(self):
         return hash((self.fname, self.subdir, self.is_built))
 
+def get_compiler_for_source(compilers, src):
+    for comp in compilers:
+        if comp.can_compile(src):
+            return comp
+    raise RuntimeError('No specified compiler can handle file {!s}'.format(src))
+
+def classify_unity_sources(compilers, sources):
+    compsrclist = {}
+    for src in sources:
+        comp = get_compiler_for_source(compilers, src)
+        if comp not in compsrclist:
+            compsrclist[comp] = [src]
+        else:
+            compsrclist[comp].append(src)
+    return compsrclist
+
 def flatten(item):
     if not isinstance(item, list):
         return item

test cases/common/110 extract same name/meson.build

@@ -1,6 +1,7 @@
 project('object extraction', 'c')
 
 lib = shared_library('somelib', ['lib.c', 'src/lib.c'])
-obj = lib.extract_objects(['lib.c', 'src/lib.c'])
+# Also tests that the object list is flattened properly
+obj = lib.extract_objects(['lib.c', ['src/lib.c']])
 exe = executable('main', 'main.c', objects: obj)
 test('extraction', exe)

test cases/common/84 extract from nested subdir/meson.build

@@ -1,4 +1,8 @@
 project('Extract objects from subdirs.', 'c')
 
-subdir('src')
-subdir('tst')
+if meson.is_unity()
+  message('Unity build: skipping incompatible test')
+else
+  subdir('src')
+  subdir('tst')
+endif

test cases/frameworks/7 gnome/mkenums/meson-sample.h

@@ -1,3 +1,5 @@
+#pragma once
+
 typedef enum
 {
     MESON_THE_XVALUE,

test cases/vala/5 target glib/meson.build

@@ -1,4 +1,8 @@
-project('valatest', 'vala', 'c', default_options : ['werror=true'])
+project('valatest', 'vala', 'c')
+
+if not meson.is_unity()
+  add_global_arguments('-Werror', language : 'c')
+endif
 
 valadeps = [dependency('glib-2.0', version : '>=2.32'), dependency('gobject-2.0')]
 

test cases/vala/8 generated sources/installed_files.txt

@@ -1 +1,2 @@
 usr/bin/generatedtest
+usr/bin/onlygentest

test cases/vala/8 generated sources/meson.build

@@ -5,3 +5,4 @@ cd.set('x', 'y')
 
 subdir('src')
 subdir('tools')
+subdir('onlygen')

test cases/vala/8 generated sources/onlygen/maingen.in

@@ -0,0 +1,3 @@
+int main() {
+    return 0;
+}

test cases/vala/8 generated sources/onlygen/meson.build

@@ -0,0 +1,7 @@
+onlygen = generator(copy,
+  output : '@BASENAME@.vala',
+  arguments : ['@INPUT@', '@OUTPUT@'])
+
+executable('onlygentest', onlygen.process('maingen.in'),
+  install : true,
+  dependencies: [dependency('glib-2.0'), dependency('gobject-2.0')])