Use listify and extract_as_list everywhere

They now flatten by default and unhold objects if required

Includes unit tests.

mesonbuild/build.py

@@ -20,7 +20,7 @@ from . import environment
 from . import dependencies
 from . import mlog
 from .mesonlib import File, MesonException, listify, extract_as_list
-from .mesonlib import flatten, typeslistify, stringlistify, classify_unity_sources
+from .mesonlib import typeslistify, stringlistify, classify_unity_sources
 from .mesonlib import get_filenames_templates_dict, substitute_values
 from .environment import for_windows, for_darwin, for_cygwin
 from .compilers import is_object, clike_langs, sort_clike, lang_suffixes
@@ -682,7 +682,7 @@ class BuildTarget(Target):
             if 'd' in self.compilers:
                 self.add_compiler_args('d', self.compilers['d'].get_feature_args(dfeatures))
 
-        self.link_args = flatten(kwargs.get('link_args', []))
+        self.link_args = extract_as_list(kwargs, 'link_args')
         for i in self.link_args:
             if not isinstance(i, str):
                 raise InvalidArguments('Link_args arguments must be strings.')
@@ -856,9 +856,7 @@ You probably should put it in link_with instead.''')
         return self.external_deps
 
     def link(self, target):
-        for t in flatten(target):
-            if hasattr(t, 'held_object'):
-                t = t.held_object
+        for t in listify(target, unholder=True):
             if not t.is_linkable_target():
                 raise InvalidArguments('Link target {!r} is not linkable.'.format(t))
             if isinstance(self, SharedLibrary) and isinstance(t, StaticLibrary) and not t.pic:
@@ -870,9 +868,7 @@ You probably should put it in link_with instead.''')
             self.link_targets.append(t)
 
     def link_whole(self, target):
-        for t in flatten(target):
-            if hasattr(t, 'held_object'):
-                t = t.held_object
+        for t in listify(target, unholder=True):
             if not isinstance(t, StaticLibrary):
                 raise InvalidArguments('{!r} is not a static library.'.format(t))
             if isinstance(self, SharedLibrary) and not t.pic:
@@ -915,7 +911,7 @@ You probably should put it in link_with instead.''')
         self.include_dirs += ids
 
     def add_compiler_args(self, language, args):
-        args = flatten(args)
+        args = listify(args)
         for a in args:
             if not isinstance(a, (str, File)):
                 raise InvalidArguments('A non-string passed to compiler args.')
@@ -1546,11 +1542,9 @@ class CustomTarget(Target):
         return deps
 
     def flatten_command(self, cmd):
-        cmd = listify(cmd)
+        cmd = listify(cmd, unholder=True)
         final_cmd = []
         for c in cmd:
-            if hasattr(c, 'held_object'):
-                c = c.held_object
             if isinstance(c, str):
                 final_cmd.append(c)
             elif isinstance(c, File):
@@ -1573,12 +1567,7 @@ class CustomTarget(Target):
 
     def process_kwargs(self, kwargs):
         super().process_kwargs(kwargs)
-        sources = flatten(kwargs.get('input', []))
-        self.sources = []
-        for s in sources:
-            if hasattr(s, 'held_object'):
-                s = s.held_object
-            self.sources.append(s)
+        self.sources = extract_as_list(kwargs, 'input', unholder=True)
         if 'output' not in kwargs:
             raise InvalidArguments('Missing keyword argument "output".')
         self.outputs = listify(kwargs['output'])

mesonbuild/dependencies/base.py

@@ -23,7 +23,7 @@ from enum import Enum
 
 from .. import mlog
 from .. import mesonlib
-from ..mesonlib import MesonException, Popen_safe, flatten, version_compare_many, listify
+from ..mesonlib import MesonException, Popen_safe, version_compare_many, listify
 
 
 # These must be defined in this file to avoid cyclical references.
@@ -586,7 +586,7 @@ class ExtraFrameworkDependency(ExternalDependency):
 
 def get_dep_identifier(name, kwargs, want_cross):
     # Need immutable objects since the identifier will be used as a dict key
-    version_reqs = flatten(kwargs.get('version', []))
+    version_reqs = listify(kwargs.get('version', []))
     if isinstance(version_reqs, list):
         version_reqs = frozenset(version_reqs)
     identifier = (name, version_reqs, want_cross)
@@ -599,7 +599,7 @@ def get_dep_identifier(name, kwargs, want_cross):
             continue
         # All keyword arguments are strings, ints, or lists (or lists of lists)
         if isinstance(value, list):
-            value = frozenset(flatten(value))
+            value = frozenset(listify(value))
         identifier += (key, value)
     return identifier
 

mesonbuild/dependencies/dev.py

@@ -21,7 +21,7 @@ import shutil
 
 from .. import mlog
 from .. import mesonlib
-from ..mesonlib import version_compare, Popen_safe
+from ..mesonlib import version_compare, Popen_safe, stringlistify, extract_as_list
 from .base import DependencyException, ExternalDependency, PkgConfigDependency
 
 class GTestDependency(ExternalDependency):
@@ -185,7 +185,7 @@ class LLVMDependency(ExternalDependency):
             raise DependencyException('Could not generate modules for LLVM.')
         self.modules = shlex.split(out)
 
-        modules = mesonlib.stringlistify(mesonlib.flatten(kwargs.get('modules', [])))
+        modules = stringlistify(extract_as_list(kwargs, 'modules'))
         for mod in sorted(set(modules)):
             if mod not in self.modules:
                 mlog.log('LLVM module', mod, 'found:', mlog.red('NO'))

mesonbuild/interpreter.py

@@ -1560,12 +1560,11 @@ class Interpreter(InterpreterBase):
         version = kwargs.get('version', self.project_version)
         if not isinstance(version, str):
             raise InterpreterException('Version must be a string.')
-        incs = extract_as_list(kwargs, 'include_directories')
-        libs = extract_as_list(kwargs, 'link_with')
+        incs = extract_as_list(kwargs, 'include_directories', unholder=True)
+        libs = extract_as_list(kwargs, 'link_with', unholder=True)
         sources = extract_as_list(kwargs, 'sources')
-        sources = self.source_strings_to_files(self.flatten(sources))
-        deps = self.flatten(kwargs.get('dependencies', []))
-        deps = listify(deps)
+        sources = listify(self.source_strings_to_files(sources), unholder=True)
+        deps = extract_as_list(kwargs, 'dependencies', unholder=True)
         compile_args = mesonlib.stringlistify(kwargs.get('compile_args', []))
         link_args = mesonlib.stringlistify(kwargs.get('link_args', []))
         final_deps = []
@@ -1577,13 +1576,8 @@ class Interpreter(InterpreterBase):
             if not isinstance(d, (dependencies.Dependency, dependencies.ExternalLibrary, dependencies.InternalDependency)):
                 raise InterpreterException('Dependencies must be external deps')
             final_deps.append(d)
-        dep = dependencies.InternalDependency(version,
-                                              mesonlib.unholder_array(incs),
-                                              compile_args,
-                                              link_args,
-                                              mesonlib.unholder_array(libs),
-                                              mesonlib.unholder_array(sources),
-                                              final_deps)
+        dep = dependencies.InternalDependency(version, incs, compile_args,
+                                              link_args, libs, sources, final_deps)
         return DependencyHolder(dep)
 
     @noKwargs
@@ -1638,7 +1632,7 @@ class Interpreter(InterpreterBase):
                                            'or not executable'.format(cmd))
             cmd = prog
         expanded_args = []
-        for a in mesonlib.flatten(cargs):
+        for a in listify(cargs):
             if isinstance(a, str):
                 expanded_args.append(a)
             elif isinstance(a, mesonlib.File):
@@ -2308,11 +2302,7 @@ to directly access options of other subprojects.''')
             raise InterpreterException('Run_target needs at least one positional argument.')
 
         cleaned_args = []
-        for i in mesonlib.flatten(all_args):
-            try:
-                i = i.held_object
-            except AttributeError:
-                pass
+        for i in listify(all_args, unholder=True):
             if not isinstance(i, (str, build.BuildTarget, build.CustomTarget, dependencies.ExternalProgram, mesonlib.File)):
                 mlog.debug('Wrong type:', str(i))
                 raise InterpreterException('Invalid argument to run_target.')
@@ -2383,11 +2373,10 @@ to directly access options of other subprojects.''')
         par = kwargs.get('is_parallel', True)
         if not isinstance(par, bool):
             raise InterpreterException('Keyword argument is_parallel must be a boolean.')
-        cmd_args = extract_as_list(kwargs, 'args')
+        cmd_args = extract_as_list(kwargs, 'args', unholder=True)
         for i in cmd_args:
-            if not isinstance(i, (str, mesonlib.File, TargetHolder)):
+            if not isinstance(i, (str, mesonlib.File, build.Target)):
                 raise InterpreterException('Command line arguments must be strings, files or targets.')
-        cmd_args = mesonlib.unholder_array(cmd_args)
         env = self.unpack_env_kwarg(kwargs)
         should_fail = kwargs.get('should_fail', False)
         if not isinstance(should_fail, bool):
@@ -2805,7 +2794,8 @@ different subdirectory.
             elif isinstance(s, str):
                 s = mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, s)
             else:
-                raise InterpreterException("Source item is not string or File-type object.")
+                raise InterpreterException('Source item is {!r} instead of '
+                                           'string or File-type object'.format(s))
             results.append(s)
         return results
 
@@ -2831,7 +2821,7 @@ different subdirectory.
         if not args:
             raise InterpreterException('Target does not have a name.')
         name = args[0]
-        sources = args[1:]
+        sources = listify(args[1:])
         if self.environment.is_cross_build():
             if kwargs.get('native', False):
                 is_cross = False
@@ -2839,19 +2829,14 @@ different subdirectory.
                 is_cross = True
         else:
             is_cross = False
-        try:
-            kw_src = self.flatten(kwargs['sources'])
-            kw_src = listify(kw_src)
-        except KeyError:
-            kw_src = []
-        sources += kw_src
+        if 'sources' in kwargs:
+            sources += listify(kwargs['sources'])
         sources = self.source_strings_to_files(sources)
-        objs = self.flatten(kwargs.get('objects', []))
-        kwargs['dependencies'] = self.flatten(kwargs.get('dependencies', []))
+        objs = extract_as_list(kwargs, 'objects')
+        kwargs['dependencies'] = extract_as_list(kwargs, 'dependencies')
         if 'extra_files' in kwargs:
             ef = extract_as_list(kwargs, 'extra_files')
             kwargs['extra_files'] = self.source_strings_to_files(ef)
-        objs = listify(objs)
         self.check_sources_exist(os.path.join(self.source_root, self.subdir), sources)
         if targetholder is ExecutableHolder:
             targetclass = build.Executable

mesonbuild/mesonlib.py

@@ -199,9 +199,6 @@ def classify_unity_sources(compilers, sources):
             compsrclist[comp].append(src)
     return compsrclist
 
-def flatten(item):
-    return listify(item, flatten=True)
-
 def is_osx():
     return platform.system().lower() == 'darwin'
 
@@ -466,34 +463,45 @@ def replace_if_different(dst, dst_tmp):
     else:
         os.unlink(dst_tmp)
 
-
-def listify(item, flatten=True):
+def listify(item, flatten=True, unholder=False):
     '''
-    Returns a list with all args embedded in a list if they are not of type list.
+    Returns a list with all args embedded in a list if they are not a list.
     This function preserves order.
+    @flatten: Convert lists of lists to a flat list
+    @unholder: Replace each item with the object it holds, if required
+
+    Note: unholding only works recursively when flattening
     '''
     if not isinstance(item, list):
+        if unholder and hasattr(item, 'held_object'):
+            item = item.held_object
         return [item]
     result = []
-    if flatten:
-        for i in item:
-            if isinstance(i, list):
-                result += listify(i, flatten=True)
-            else:
-                result.append(i)
-    else:
-        for i in item:
+    for i in item:
+        if unholder and hasattr(i, 'held_object'):
+            i = i.held_object
+        if flatten and isinstance(i, list):
+            result += listify(i, flatten=True, unholder=unholder)
+        else:
             result.append(i)
     return result
 
 
-def extract_as_list(dict_object, *keys, pop=False):
+def extract_as_list(dict_object, *keys, pop=False, **kwargs):
     '''
     Extracts all values from given dict_object and listifies them.
     '''
+    result = []
+    fetch = dict_object.get
     if pop:
-        return flatten([dict_object.pop(key, []) for key in keys])
-    return flatten([dict_object.get(key, []) for key in keys])
+        fetch = dict_object.pop
+    # If there's only one key, we don't return a list with one element
+    if len(keys) == 1:
+        return listify(fetch(keys[0], []), **kwargs)
+    # Return a list of values corresponding to *keys
+    for key in keys:
+        result.append(listify(fetch(key, []), **kwargs))
+    return result
 
 
 def typeslistify(item, types):
@@ -752,15 +760,6 @@ def windows_proof_rmtree(f):
     # Try one last time and throw if it fails.
     shutil.rmtree(f)
 
-def unholder_array(entries):
-    result = []
-    entries = flatten(entries)
-    for e in entries:
-        if hasattr(e, 'held_object'):
-            e = e.held_object
-        result.append(e)
-    return result
-
 class OrderedSet(collections.MutableSet):
     """A set that preserves the order in which items are added, by first
     insertion.

mesonbuild/modules/gnome.py

@@ -20,7 +20,7 @@ import os
 import copy
 import subprocess
 from . import ModuleReturnValue
-from ..mesonlib import MesonException, OrderedSet, unholder_array, Popen_safe
+from ..mesonlib import MesonException, OrderedSet, Popen_safe, extract_as_list
 from ..dependencies import Dependency, PkgConfigDependency, InternalDependency
 from .. import mlog
 from .. import mesonlib
@@ -323,7 +323,7 @@ class GnomeModule(ExtensionModule):
         cflags = OrderedSet()
         ldflags = OrderedSet()
         gi_includes = OrderedSet()
-        deps = unholder_array(deps)
+        deps = mesonlib.listify(deps, unholder=True)
 
         for dep in deps:
             if isinstance(dep, InternalDependency):
@@ -415,7 +415,7 @@ class GnomeModule(ExtensionModule):
             raise MesonException('gobject-introspection dependency was not found, gir cannot be generated.')
         ns = kwargs.pop('namespace')
         nsversion = kwargs.pop('nsversion')
-        libsources = mesonlib.flatten(kwargs.pop('sources'))
+        libsources = mesonlib.extract_as_list(kwargs, 'sources', pop=True)
         girfile = '%s-%s.gir' % (ns, nsversion)
         srcdir = os.path.join(state.environment.get_source_dir(), state.subdir)
         builddir = os.path.join(state.environment.get_build_dir(), state.subdir)
@@ -524,7 +524,7 @@ class GnomeModule(ExtensionModule):
                 raise MesonException('Gir export packages must be str or list')
 
         deps = (girtarget.get_all_link_deps() + girtarget.get_external_deps() +
-                unholder_array(kwargs.pop('dependencies', [])))
+                extract_as_list(kwargs, 'dependencies', pop=True, unholder=True))
         # Need to recursively add deps on GirTarget sources from our
         # dependencies and also find the include directories needed for the
         # typelib generation custom target below.
@@ -791,7 +791,7 @@ This will become a hard error in the future.''')
 
     def _get_build_args(self, kwargs, state):
         args = []
-        deps = unholder_array(kwargs.get('dependencies', []))
+        deps = extract_as_list(kwargs, 'dependencies', unholder=True)
         cflags, ldflags, gi_includes = self._get_dependencies_flags(deps, state, include_rpath=True)
         inc_dirs = mesonlib.extract_as_list(kwargs, 'include_directories')
         for incd in inc_dirs:

run_unittests.py

@@ -31,6 +31,7 @@ import mesonbuild.compilers
 import mesonbuild.environment
 import mesonbuild.mesonlib
 import mesonbuild.coredata
+from mesonbuild.interpreter import ObjectHolder
 from mesonbuild.mesonlib import is_linux, is_windows, is_osx, is_cygwin, windows_proof_rmtree
 from mesonbuild.environment import Environment
 from mesonbuild.dependencies import DependencyException
@@ -62,7 +63,6 @@ def get_soname(fname):
 def get_rpath(fname):
     return get_dynamic_section_entry(fname, r'(?:rpath|runpath)')
 
-
 class InternalTests(unittest.TestCase):
 
     def test_version_number(self):
@@ -398,6 +398,49 @@ class InternalTests(unittest.TestCase):
 
         self.assertEqual(forced_value, desired_value)
 
+    def test_listify(self):
+        listify = mesonbuild.mesonlib.listify
+        # Test sanity
+        self.assertEqual([1], listify(1))
+        self.assertEqual([], listify([]))
+        self.assertEqual([1], listify([1]))
+        # Test flattening
+        self.assertEqual([1, 2, 3], listify([1, [2, 3]]))
+        self.assertEqual([1, 2, 3], listify([1, [2, [3]]]))
+        self.assertEqual([1, [2, [3]]], listify([1, [2, [3]]], flatten=False))
+        # Test flattening and unholdering
+        holder1 = ObjectHolder(1)
+        holder3 = ObjectHolder(3)
+        self.assertEqual([holder1], listify(holder1))
+        self.assertEqual([holder1], listify([holder1]))
+        self.assertEqual([holder1, 2], listify([holder1, 2]))
+        self.assertEqual([holder1, 2, 3], listify([holder1, 2, [3]]))
+        self.assertEqual([1], listify(holder1, unholder=True))
+        self.assertEqual([1], listify([holder1], unholder=True))
+        self.assertEqual([1, 2], listify([holder1, 2], unholder=True))
+        self.assertEqual([1, 2, 3], listify([holder1, 2, [holder3]], unholder=True))
+        # Unholding doesn't work recursively when not flattening
+        self.assertEqual([1, [2], [holder3]], listify([holder1, [2], [holder3]], unholder=True, flatten=False))
+
+    def test_extract_as_list(self):
+        extract = mesonbuild.mesonlib.extract_as_list
+        # Test sanity
+        kwargs = {'sources': [1, 2, 3]}
+        self.assertEqual([1, 2, 3], extract(kwargs, 'sources'))
+        self.assertEqual(kwargs, {'sources': [1, 2, 3]})
+        self.assertEqual([1, 2, 3], extract(kwargs, 'sources', pop=True))
+        self.assertEqual(kwargs, {})
+        # Test unholding
+        holder3 = ObjectHolder(3)
+        kwargs = {'sources': [1, 2, holder3]}
+        self.assertEqual([1, 2, 3], extract(kwargs, 'sources', unholder=True))
+        self.assertEqual(kwargs, {'sources': [1, 2, holder3]})
+        self.assertEqual([1, 2, 3], extract(kwargs, 'sources', unholder=True, pop=True))
+        self.assertEqual(kwargs, {})
+        # Test listification
+        kwargs = {'sources': [1, 2, 3], 'pch_sources': [4, 5, 6]}
+        self.assertEqual([[1, 2, 3], [4, 5, 6]], extract(kwargs, 'sources', 'pch_sources'))
+
 
 class BasePlatformTests(unittest.TestCase):
     def setUp(self):