Add support for extracting objects in unity builds

Not only does extract_all_objects() now work properly again,
extract_objects() also works if you specify a subset of sources all of
which have been compiled into a single unified object.

So, for instance, this allows you to extract all the objects
corresponding to the C sources compiled into a target consisting of
C and C++ sources.

mesonbuild/backend/backends.py

@@ -19,7 +19,7 @@ from .. import mesonlib
 from .. import compilers
 import json
 import subprocess
-from ..mesonlib import MesonException
+from ..mesonlib import MesonException, get_compiler_for_source, classify_unity_sources
 
 class InstallData():
     def __init__(self, source_dir, build_dir, prefix):
@@ -78,21 +78,6 @@ class Backend():
             priv_dirname = self.get_target_private_dir_abs(t)
             os.makedirs(priv_dirname, exist_ok=True)
 
-    def get_compiler_for_lang(self, lang):
-        for i in self.build.compilers:
-            if i.language == lang:
-                return i
-        raise RuntimeError('No compiler for language ' + lang)
-
-    def get_compiler_for_source(self, src, is_cross):
-        comp = self.build.cross_compilers if is_cross else self.build.compilers
-        for i in comp:
-            if i.can_compile(src):
-                return i
-        if isinstance(src, mesonlib.File):
-            src = src.fname
-        raise RuntimeError('No specified compiler can handle file ' + src)
-
     def get_target_filename(self, t):
         if isinstance(t, build.CustomTarget):
             if len(t.get_outputs()) != 1:
@@ -153,14 +138,17 @@ class Backend():
         # target that the GeneratedList is used in
         return os.path.join(self.get_target_private_dir(target), src)
 
+    def get_unity_source_filename(self, target, suffix):
+        return target.name + '-unity.' + suffix
+
     def generate_unity_files(self, target, unity_src):
-        langlist = {}
         abs_files = []
         result = []
+        compsrcs = classify_unity_sources(target.compilers.values(), unity_src)
 
-        def init_language_file(language, suffix):
+        def init_language_file(suffix):
             outfilename = os.path.join(self.get_target_private_dir_abs(target),
-                                       target.name + '-unity' + suffix)
+                                       self.get_unity_source_filename(target, suffix))
             outfileabs = os.path.join(self.environment.get_build_dir(),
                                       outfilename)
             outfileabs_tmp = outfileabs + '.tmp'
@@ -171,20 +159,12 @@ class Backend():
             result.append(outfilename)
             return open(outfileabs_tmp, 'w')
 
-        try:
-            for src in unity_src:
-                comp = self.get_compiler_for_source(src, target.is_cross)
-                language = comp.get_language()
-                try:
-                    ofile = langlist[language]
-                except KeyError:
-                    suffix = '.' + comp.get_default_suffix()
-                    ofile = langlist[language] = init_language_file(language,
-                                                                    suffix)
-                ofile.write('#include<%s>\n' % src)
-        finally:
-            for x in langlist.values():
-                x.close()
+        # For each language, generate a unity source file and return the list
+        for comp, srcs in compsrcs.items():
+            lang = comp.get_language()
+            with init_language_file(comp.get_default_suffix()) as ofile:
+                for src in srcs:
+                    ofile.write('#include<%s>\n' % src)
         [mesonlib.replace_if_different(x, x + '.tmp') for x in abs_files]
         return result
 
@@ -281,21 +261,34 @@ class Backend():
         raise AssertionError("BUG: Couldn't determine linker for sources {!r}".format(src))
 
     def object_filename_from_source(self, target, source):
-        return source.fname.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
+        if isinstance(source, mesonlib.File):
+            source = source.fname
+        return source.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
 
-    def determine_ext_objs(self, extobj, proj_dir_to_build_root=''):
+    def determine_ext_objs(self, extobj, proj_dir_to_build_root):
         result = []
         targetdir = self.get_target_private_dir(extobj.target)
+        # With unity builds, there's just one object that contains all the
+        # sources, so if we want all the objects, just return that.
+        if self.environment.coredata.get_builtin_option('unity'):
+            if not extobj.unity_compatible:
+                # This should never happen
+                msg = 'BUG: Meson must not allow extracting single objects ' \
+                      'in Unity builds'
+                raise AssertionError(msg)
+            comp = get_compiler_for_source(extobj.target.compilers.values(),
+                                           extobj.srclist[0])
+            # The unity object name uses the full absolute path of the source file
+            osrc = os.path.join(self.get_target_private_dir_abs(extobj.target),
+                                self.get_unity_source_filename(extobj.target,
+                                                               comp.get_default_suffix()))
+            objname = self.object_filename_from_source(extobj.target, osrc)
+            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
+            return [objpath]
         for osrc in extobj.srclist:
-            # If extracting in a subproject, the subproject
-            # name gets duplicated in the file name.
-            pathsegs = osrc.subdir.split(os.sep)
-            if pathsegs[0] == 'subprojects':
-                pathsegs = pathsegs[2:]
-            fixedpath = os.sep.join(pathsegs)
-            objname = os.path.join(proj_dir_to_build_root, targetdir,
-                                   self.object_filename_from_source(extobj.target, osrc))
-            result.append(objname)
+            objname = self.object_filename_from_source(extobj.target, osrc)
+            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
+            result.append(objpath)
         return result
 
     def get_pch_include_args(self, compiler, target):

mesonbuild/backend/ninjabackend.py

@@ -18,7 +18,7 @@ from .. import build
 from .. import mlog
 from .. import dependencies
 from .. import compilers
-from ..mesonlib import File, MesonException
+from ..mesonlib import File, MesonException, get_compiler_for_source
 from .backends import InstallData
 from ..build import InvalidArguments
 import os, sys, pickle, re
@@ -1684,7 +1684,7 @@ rule FORTRAN_DEP_HACK
         if isinstance(src, RawFilename) and src.fname.endswith('.h'):
             raise AssertionError('BUG: sources should not contain headers')
         extra_orderdeps = []
-        compiler = self.get_compiler_for_source(src, target.is_cross)
+        compiler = get_compiler_for_source(target.compilers.values(), src)
         commands = []
         # The first thing is implicit include directories: source, build and private.
         commands += compiler.get_include_args(self.get_target_private_dir(target), False)
@@ -1867,7 +1867,7 @@ rule FORTRAN_DEP_HACK
                       'directory as source, please put it in a subdirectory.' \
                       ''.format(target.get_basename())
                 raise InvalidArguments(msg)
-            compiler = self.get_compiler_for_lang(lang)
+            compiler = target.compilers[lang]
             if compiler.id == 'msvc':
                 src = os.path.join(self.build_to_src, target.get_source_subdir(), pch[-1])
                 (commands, dep, dst, objs) = self.generate_msvc_pch_command(target, compiler, pch)

mesonbuild/build.py

@@ -17,7 +17,7 @@ from . import environment
 from . import dependencies
 from . import mlog
 import copy, os, re
-from .mesonlib import File, flatten, MesonException, stringlistify
+from .mesonlib import File, flatten, MesonException, stringlistify, classify_unity_sources
 from .environment import for_windows, for_darwin
 
 known_basic_kwargs = {'install' : True,
@@ -176,9 +176,42 @@ class IncludeDirs():
         return self.extra_build_dirs
 
 class ExtractedObjects():
+    '''
+    Holds a list of sources for which the objects must be extracted
+    '''
     def __init__(self, target, srclist):
         self.target = target
         self.srclist = srclist
+        self.check_unity_compatible()
+
+    def check_unity_compatible(self):
+        # Figure out if the extracted object list is compatible with a Unity
+        # build. When we're doing a Unified build, we go through the sources,
+        # and create a single source file from each subset of the sources that
+        # can be compiled with a specific compiler. Then we create one object
+        # from each unified source file.
+        # If the list of sources for which we want objects is the same as the
+        # list of sources that go into each unified build, we're good.
+        self.unity_compatible = False
+        srclist_set = set(self.srclist)
+        # Objects for all the sources are required, so we're compatible
+        if srclist_set == set(self.target.sources):
+            self.unity_compatible = True
+            return
+        # Check if the srclist is a subset (of the target's sources) that is
+        # going to form a unified source file and a single object
+        compsrcs = classify_unity_sources(self.target.compilers.values(),
+                                          self.target.sources)
+        for srcs in compsrcs.values():
+            if srclist_set == set(srcs):
+                self.unity_compatible = True
+                return
+        msg = 'Single object files can not be extracted in Unity builds. ' \
+              'You can only extract all the object files at once.'
+        raise MesonException(msg)
+
+    def get_want_all_objects(self):
+        return self.want_all_objects
 
 class EnvironmentVariables():
     def __init__(self):
@@ -397,18 +430,15 @@ class BuildTarget():
         if 'link_with' in self.kwargs:
             self.kwargs['link_with'] = self.unpack_holder(self.kwargs['link_with'])
 
-    def extract_objects(self, srcargs):
+    def extract_objects(self, srclist):
         obj_src = []
-        for srclist in srcargs:
-            if not isinstance(srclist, list):
-                srclist = [srclist]
-            for src in srclist:
-                if not isinstance(src, str):
-                    raise MesonException('Extraction arguments must be strings.')
-                src = File(False, self.subdir, src)
-                if src not in self.sources:
-                    raise MesonException('Tried to extract unknown source %s.' % src)
-                obj_src.append(src)
+        for src in srclist:
+            if not isinstance(src, str):
+                raise MesonException('Object extraction arguments must be strings.')
+            src = File(False, self.subdir, src)
+            if src not in self.sources:
+                raise MesonException('Tried to extract unknown source %s.' % src)
+            obj_src.append(src)
         return ExtractedObjects(self, obj_src)
 
     def extract_all_objects(self):

mesonbuild/interpreter.py

@@ -2571,8 +2571,6 @@ requirements use the version keyword argument instead.''')
         else:
             obj = self.evaluate_statement(invokable)
         method_name = node.name
-        if method_name == 'extract_objects' and self.environment.coredata.get_builtin_option('unity'):
-            raise InterpreterException('Single object files can not be extracted in Unity builds.')
         args = node.args
         if isinstance(obj, mparser.StringNode):
             obj = obj.get_value()

mesonbuild/mesonlib.py

@@ -70,6 +70,22 @@ class File:
     def __hash__(self):
         return hash((self.fname, self.subdir, self.is_built))
 
+def get_compiler_for_source(compilers, src):
+    for comp in compilers:
+        if comp.can_compile(src):
+            return comp
+    raise RuntimeError('No specified compiler can handle file {!s}'.format(src))
+
+def classify_unity_sources(compilers, sources):
+    compsrclist = {}
+    for src in sources:
+        comp = get_compiler_for_source(compilers, src)
+        if comp not in compsrclist:
+            compsrclist[comp] = [src]
+        else:
+            compsrclist[comp].append(src)
+    return compsrclist
+
 def flatten(item):
     if not isinstance(item, list):
         return item

test cases/common/110 extract same name/meson.build

@@ -1,6 +1,7 @@
 project('object extraction', 'c')
 
 lib = shared_library('somelib', ['lib.c', 'src/lib.c'])
-obj = lib.extract_objects(['lib.c', 'src/lib.c'])
+# Also tests that the object list is flattened properly
+obj = lib.extract_objects(['lib.c', ['src/lib.c']])
 exe = executable('main', 'main.c', objects: obj)
 test('extraction', exe)

test cases/common/84 extract from nested subdir/meson.build

@@ -1,4 +1,8 @@
 project('Extract objects from subdirs.', 'c')
 
-subdir('src')
-subdir('tst')
+if meson.is_unity()
+  message('Unity build: skipping incompatible test')
+else
+  subdir('src')
+  subdir('tst')
+endif

test cases/frameworks/7 gnome/mkenums/meson-sample.h

@@ -1,3 +1,5 @@
+#pragma once
+
 typedef enum
 {
     MESON_THE_XVALUE,

test cases/vala/5 target glib/meson.build

@@ -1,4 +1,8 @@
-project('valatest', 'vala', 'c', default_options : ['werror=true'])
+project('valatest', 'vala', 'c')
+
+if not meson.is_unity()
+  add_global_arguments('-Werror', language : 'c')
+endif
 
 valadeps = [dependency('glib-2.0', version : '>=2.32'), dependency('gobject-2.0')]