Merge pull request #3404 from xclaesse/extract-recursive

extract_all_objects(): Recursively extract objects

docs/markdown/Reference-manual.md

@@ -1736,7 +1736,11 @@ A build target is either an [executable](#executable),
 [shared module](#shared_module).
 
 - `extract_all_objects()` is same as `extract_objects` but returns all
-  object files generated by this target
+  object files generated by this target. Since 0.46.0 keyword argument
+  `recursive` must be set to `true` to also return objects passed to the
+  `object` argument of this target. By default only objects built for this
+  target are returned to maintain backward compatibility with previous versions.
+  The default will eventually be changed to `true` in a future version.
 
 - `extract_objects()` returns an opaque value representing the
   generated object files of arguments, usually used to take single

docs/markdown/snippets/extract-all-objects.md

@@ -0,0 +1,12 @@
+## Recursively extract objects
+
+`recursive` keyword argument has been added to `extract_all_objects`. When set
+to `true` it will also return objects passed to the `objects` argument of this
+target. By default only objects built for this target are returned to maintain
+backward compatibility with previous versions. The default will eventually be
+changed to `true` in a future version.
+
+```meson
+lib1 = static_library('a', 'source.c', objects : 'prebuilt.o')
+lib2 = static_library('b', objects : lib1.extract_all_objects(recursive : true))
+```

mesonbuild/backend/backends.py

@@ -206,6 +206,8 @@ class Backend:
         return os.path.join(self.get_target_private_dir(target), src)
 
     def get_unity_source_file(self, target, suffix):
+        # There is a potential conflict here, but it is unlikely that
+        # anyone both enables unity builds and has a file called foo-unity.cpp.
         osrc = target.name + '-unity.' + suffix
         return mesonlib.File.from_built_file(self.get_target_private_dir(target), osrc)
 
@@ -239,8 +241,11 @@ class Backend:
                                os.path.join('dummyprefixdir', fromdir))
 
     def flatten_object_list(self, target, proj_dir_to_build_root=''):
+        return self._flatten_object_list(target, target.get_objects(), proj_dir_to_build_root)
+
+    def _flatten_object_list(self, target, objects, proj_dir_to_build_root):
         obj_list = []
-        for obj in target.get_objects():
+        for obj in objects:
             if isinstance(obj, str):
                 o = os.path.join(proj_dir_to_build_root,
                                  self.build_to_src, target.get_subdir(), obj)
@@ -248,7 +253,9 @@ class Backend:
             elif isinstance(obj, mesonlib.File):
                 obj_list.append(obj.rel_to_builddir(self.build_to_src))
             elif isinstance(obj, build.ExtractedObjects):
-                obj_list += self.determine_ext_objs(target, obj, proj_dir_to_build_root)
+                if obj.recursive:
+                    obj_list += self._flatten_object_list(obj.target, obj.objlist, proj_dir_to_build_root)
+                obj_list += self.determine_ext_objs(obj, proj_dir_to_build_root)
             else:
                 raise MesonException('Unknown data type in object list.')
         return obj_list
@@ -361,15 +368,11 @@ class Backend:
                 result += [rp]
         return result
 
-    def object_filename_from_source(self, target, source, is_unity):
+    def object_filename_from_source(self, target, source):
         assert isinstance(source, mesonlib.File)
         build_dir = self.environment.get_build_dir()
         rel_src = source.rel_to_builddir(self.build_to_src)
 
-        if (not self.environment.is_source(rel_src) or
-           self.environment.is_header(rel_src)) and not is_unity:
-            return None
-
         # foo.vala files compile down to foo.c and then foo.c.o, not foo.vala.o
         if rel_src.endswith(('.vala', '.gs')):
             # See description in generate_vala_compile for this logic.
@@ -379,8 +382,6 @@ class Backend:
                 rel_src = os.path.relpath(rel_src, self.get_target_private_dir(target))
             else:
                 rel_src = os.path.basename(rel_src)
-            if is_unity:
-                return 'meson-generated_' + rel_src[:-5] + '.c.' + self.environment.get_object_suffix()
             # A meson- prefixed directory is reserved; hopefully no-one creates a file name with such a weird prefix.
             source = 'meson-generated_' + rel_src[:-5] + '.c'
         elif source.is_built:
@@ -398,24 +399,10 @@ class Backend:
                                          os.path.join(self.environment.get_source_dir(), target.get_subdir()))
         return source.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
 
-    def determine_ext_objs(self, target, extobj, proj_dir_to_build_root):
+    def determine_ext_objs(self, extobj, proj_dir_to_build_root):
         result = []
-        targetdir = self.get_target_private_dir(extobj.target)
-        # With unity builds, there's just one object that contains all the
-        # sources, and we only support extracting all the objects in this mode,
-        # so just return that.
-        if self.is_unity(target):
-            comp = get_compiler_for_source(extobj.target.compilers.values(),
-                                           extobj.srclist[0])
-            # There is a potential conflict here, but it is unlikely that
-            # anyone both enables unity builds and has a file called foo-unity.cpp.
-            osrc = self.get_unity_source_file(extobj.target,
-                                              comp.get_default_suffix())
-            objname = self.object_filename_from_source(extobj.target, osrc, True)
-            objname = objname.replace('/', '_').replace('\\', '_')
-            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
-            return [objpath]
 
+        # Merge sources and generated sources
         sources = list(extobj.srclist)
         for gensrc in extobj.genlist:
             for s in gensrc.get_outputs():
@@ -423,11 +410,30 @@ class Backend:
                 dirpart, fnamepart = os.path.split(path)
                 sources.append(File(True, dirpart, fnamepart))
 
+        # Filter out headers and all non-source files
+        sources = [s for s in sources if self.environment.is_source(s) and not self.environment.is_header(s)]
+
+        # extobj could contain only objects and no sources
+        if not sources:
+            return result
+
+        targetdir = self.get_target_private_dir(extobj.target)
+
+        # With unity builds, there's just one object that contains all the
+        # sources, and we only support extracting all the objects in this mode,
+        # so just return that.
+        if self.is_unity(extobj.target):
+            compsrcs = classify_unity_sources(extobj.target.compilers.values(), sources)
+            sources = []
+            for comp in compsrcs.keys():
+                osrc = self.get_unity_source_file(extobj.target,
+                                                  comp.get_default_suffix())
+                sources.append(osrc)
+
         for osrc in sources:
-            objname = self.object_filename_from_source(extobj.target, osrc, False)
+            objname = self.object_filename_from_source(extobj.target, osrc)
-            if objname:
+            objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
-                objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
+            result.append(objpath)
-                result.append(objpath)
 
         return result
 

mesonbuild/backend/ninjabackend.py

@@ -2206,7 +2206,7 @@ rule FORTRAN_DEP_HACK
             raise AssertionError('BUG: broken generated source file handling for {!r}'.format(src))
         else:
             raise InvalidArguments('Invalid source type: {!r}'.format(src))
-        obj_basename = self.object_filename_from_source(target, src, self.is_unity(target))
+        obj_basename = self.object_filename_from_source(target, src)
         rel_obj = os.path.join(self.get_target_private_dir(target), obj_basename)
         dep_file = compiler.depfile_for_object(rel_obj)
 

mesonbuild/backend/vs2010backend.py

@@ -1026,7 +1026,7 @@ class Vs2010Backend(backends.Backend):
                 self.add_additional_options(lang, inc_cl, file_args)
                 self.add_preprocessor_defines(lang, inc_cl, file_defines)
                 self.add_include_dirs(lang, inc_cl, file_inc_dirs)
-                ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + self.object_filename_from_source(target, s, False)
+                ET.SubElement(inc_cl, 'ObjectFileName').text = "$(IntDir)" + self.object_filename_from_source(target, s)
             for s in gen_src:
                 inc_cl = ET.SubElement(inc_src, 'CLCompile', Include=s)
                 lang = Vs2010Backend.lang_from_source_file(s)

mesonbuild/build.py

@@ -213,40 +213,49 @@ class ExtractedObjects:
     '''
     Holds a list of sources for which the objects must be extracted
     '''
-    def __init__(self, target, srclist, genlist, is_unity):
+    def __init__(self, target, srclist=[], genlist=[], objlist=[], recursive=True):
         self.target = target
+        self.recursive = recursive
         self.srclist = srclist
         self.genlist = genlist
-        if is_unity:
+        self.objlist = objlist
+        if self.target.is_unity:
             self.check_unity_compatible()
 
     def __repr__(self):
         r = '<{0} {1!r}: {2}>'
         return r.format(self.__class__.__name__, self.target.name, self.srclist)
 
+    def classify_all_sources(self, sources, generated_sources):
+        # Merge sources and generated sources
+        sources = list(sources)
+        for gensrc in generated_sources:
+            for s in gensrc.get_outputs():
+                # We cannot know the path where this source will be generated,
+                # but all we need here is the file extension to determine the
+                # compiler.
+                sources.append(s)
+
+        # Filter out headers and all non-source files
+        sources = [s for s in sources if environment.is_source(s) and not environment.is_header(s)]
+
+        return classify_unity_sources(self.target.compilers.values(), sources)
+
     def check_unity_compatible(self):
         # Figure out if the extracted object list is compatible with a Unity
         # build. When we're doing a Unified build, we go through the sources,
         # and create a single source file from each subset of the sources that
         # can be compiled with a specific compiler. Then we create one object
-        # from each unified source file.
+        # from each unified source file. So for each compiler we can either
-        # If the list of sources for which we want objects is the same as the
+        # extra all its sources or none.
-        # list of sources that go into each unified build, we're good.
+        cmpsrcs = self.classify_all_sources(self.target.sources, self.target.generated)
-        srclist_set = set(self.srclist)
+        extracted_cmpsrcs = self.classify_all_sources(self.srclist, self.genlist)
-        # Objects for all the sources are required, so we're compatible
-        if srclist_set == set(self.target.sources):
-            return
-        # Check if the srclist is a subset (of the target's sources) that is
-        # going to form a unified source file and a single object
-        compsrcs = classify_unity_sources(self.target.compilers.values(),
-                                          self.target.sources)
-        for srcs in compsrcs.values():
-            if srclist_set == set(srcs):
-                return
-        msg = 'Single object files can not be extracted in Unity builds. ' \
-              'You can only extract all the object files at once.'
-        raise MesonException(msg)
 
+        for comp, srcs in extracted_cmpsrcs.items():
+            if set(srcs) != set(cmpsrcs[comp]):
+                raise MesonException('Single object files can not be extracted '
+                                     'in Unity builds. You can only extract all '
+                                     'the object files for each compiler at once.')
 
 class EnvironmentVariables:
     def __init__(self):
@@ -637,13 +646,11 @@ class BuildTarget(Target):
             if src not in self.sources:
                 raise MesonException('Tried to extract unknown source %s.' % src)
             obj_src.append(src)
-        return ExtractedObjects(self, obj_src, [], self.is_unity)
+        return ExtractedObjects(self, obj_src)
 
-    def extract_all_objects(self):
+    def extract_all_objects(self, recursive=True):
-        # FIXME: We should add support for transitive extract_objects()
+        return ExtractedObjects(self, self.sources, self.generated, self.objects,
-        if self.objects:
+                                recursive)
-            raise MesonException('Cannot extract objects from a target that itself has extracted objects')
-        return ExtractedObjects(self, self.sources, self.generated, self.is_unity)
 
     def get_all_link_deps(self):
         return self.get_transitive_link_deps()

mesonbuild/interpreter.py

@@ -606,8 +606,15 @@ class BuildTargetHolder(TargetHolder):
         gobjs = self.held_object.extract_objects(args)
         return GeneratedObjectsHolder(gobjs)
 
+    @permittedMethodKwargs({'recursive'})
     def extract_all_objects_method(self, args, kwargs):
-        gobjs = self.held_object.extract_all_objects()
+        recursive = kwargs.get('recursive', False)
+        gobjs = self.held_object.extract_all_objects(recursive)
+        if gobjs.objlist and 'recursive' not in kwargs:
+            mlog.warning('extract_all_objects called without setting recursive '
+                         'keyword argument. Meson currently defaults to '
+                         'non-recursive to maintain backward compatibility but '
+                         'the default will be changed in the future.')
         return GeneratedObjectsHolder(gobjs)
 
     def get_id_method(self, args, kwargs):

test cases/common/88 extract all/meson.build

@@ -2,8 +2,12 @@ project('extract all', 'c')
 
 a = static_library('a', 'one.c', 'two.c')
 b = static_library('b', 'three.c', 'four.c')
-c = static_library('c', 
+c = static_library('c', objects : [a.extract_all_objects(), b.extract_all_objects()])
-  objects : [a.extract_all_objects(), b.extract_all_objects()])
+d = static_library('d', objects : [a.extract_all_objects(), b.extract_all_objects(), c.extract_all_objects()])
+d_recursive = static_library('d_recursive', objects : [c.extract_all_objects(recursive : true)])
 
-e = executable('proggie', 'prog.c', link_with : c)
+e = executable('proggie', 'prog.c', link_with : d)
 test('extall', e)
+
+e = executable('proggie_recursive', 'prog.c', link_with : d_recursive)
+test('extall_recursive', e)