Merge wraps from subprojects into wraps from main project

wraps from subprojects are now merged into the list of wraps from main
project, so they can be used to download dependencies of dependencies
instead of having to promote wraps manually. If multiple projects
provides the same wrap file, the first one to be configured wins.

This also fix usage of sub-subproject that don't have wrap files. We can
now configure B when its source tree is at
`subprojects/A/subprojects/B/`. This has the implication that we cannot
assume that subproject "foo" is at `self.subproject_dir / 'foo'` any
more.

docs/markdown/Using-wraptool.md

@@ -76,24 +76,3 @@ straightforward:
 Wraptool can do other things besides these. Documentation for these
 can be found in the command line help, which can be accessed by
 `meson wrap --help`.
-
-## Promoting dependencies
-
-Meson will only search for subprojects from the top level
-`subprojects` directory. If you have subprojects that themselves have
-subprojects, you must transfer them to the top level. This can be done
-by going to your source root and issuing a promotion command.
-
-    meson wrap promote projname
-
-This will cause Meson to go through your entire project tree, find an
-embedded subproject and copy it to the top level.
-
-If there are multiple embedded copies of a subproject, Meson will not
-try to guess which one you want. Instead it will print all the
-possibilities. You can then manually select which one to promote by
-writing it out fully.
-
-    meson wrap promote subprojects/s1/subprojects/projname
-
-This functionality was added in Meson release 0.45.0.

docs/markdown/snippets/subsubproject.md

@@ -0,0 +1,11 @@
+## Sub-subprojects
+
+It is not required to promote wrap files for subprojects into the main project
+any more. When configuring a subproject, meson will look for any wrap file or
+directory in the subproject's `subprojects/` directory and add them into the
+global list of available subprojects, to be used by any future `subproject()`
+call or `dependency()` fallback. If a subproject with the same name already exists,
+the new wrap file or directory is ignored. That means that the main project can
+always override any subproject's wrap files by providing their own, it also means
+the ordering in which subprojects are configured matters, if 2 subprojects provide
+foo.wrap only the one from the first subproject to be configured will be used.

mesonbuild/interpreter.py

@@ -1010,15 +1010,14 @@ class Test(InterpreterObject):
 
 class SubprojectHolder(InterpreterObject, ObjectHolder):
 
-    def __init__(self, subinterpreter, subproject_dir, name, warnings=0, disabled_feature=None,
+    def __init__(self, subinterpreter, subdir, warnings=0, disabled_feature=None,
                  exception=None):
         InterpreterObject.__init__(self)
         ObjectHolder.__init__(self, subinterpreter)
-        self.name = name
         self.warnings = warnings
         self.disabled_feature = disabled_feature
         self.exception = exception
-        self.subproject_dir = subproject_dir
+        self.subdir = PurePath(subdir).as_posix()
         self.methods.update({'get_variable': self.get_variable_method,
                              'found': self.found_method,
                              })
@@ -1037,8 +1036,7 @@ class SubprojectHolder(InterpreterObject, ObjectHolder):
         if len(args) < 1 or len(args) > 2:
             raise InterpreterException('Get_variable takes one or two arguments.')
         if not self.found():
-            raise InterpreterException('Subproject "%s/%s" disabled can\'t get_variable on it.' % (
-                self.subproject_dir, self.name))
+            raise InterpreterException('Subproject "%s" disabled can\'t get_variable on it.' % (self.subdir))
         varname = args[0]
         if not isinstance(varname, str):
             raise InterpreterException('Get_variable first argument must be a string.')
@@ -2844,7 +2842,7 @@ external dependencies (including libraries) must go to "dependencies".''')
         return self.do_subproject(subp_name, 'meson', kwargs)
 
     def disabled_subproject(self, subp_name, disabled_feature=None, exception=None):
-        sub = SubprojectHolder(None, self.subproject_dir, subp_name,
+        sub = SubprojectHolder(None, os.path.join(self.subproject_dir, subp_name),
                                disabled_feature=disabled_feature, exception=exception)
         self.subprojects[subp_name] = sub
         return sub
@@ -2882,28 +2880,19 @@ external dependencies (including libraries) must go to "dependencies".''')
         if subp_name in self.subprojects:
             subproject = self.subprojects[subp_name]
             if required and not subproject.found():
-                raise InterpreterException('Subproject "%s/%s" required but not found.' % (
-                                           self.subproject_dir, subp_name))
+                raise InterpreterException('Subproject "%s" required but not found.' % (subproject.subdir))
             return subproject
 
         r = self.environment.wrap_resolver
         try:
-            resolved = r.resolve(subp_name, method, self.subproject)
+            subdir = r.resolve(subp_name, method, self.subproject)
         except wrap.WrapException as e:
-            subprojdir = os.path.join(self.subproject_dir, r.directory)
-            if isinstance(e, wrap.WrapNotFoundException):
-                # if the reason subproject execution failed was because
-                # the directory doesn't exist, try to give some helpful
-                # advice if it's a nested subproject that needs
-                # promotion...
-                self.print_nested_info(subp_name)
             if not required:
                 mlog.log(e)
-                mlog.log('Subproject ', mlog.bold(subprojdir), 'is buildable:', mlog.red('NO'), '(disabling)')
+                mlog.log('Subproject ', mlog.bold(subp_name), 'is buildable:', mlog.red('NO'), '(disabling)')
                 return self.disabled_subproject(subp_name, exception=e)
             raise e
 
-        subdir = os.path.join(self.subproject_dir, resolved)
         subdir_abs = os.path.join(self.environment.get_source_dir(), subdir)
         os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True)
         self.global_args_frozen = True
@@ -2927,7 +2916,7 @@ external dependencies (including libraries) must go to "dependencies".''')
                     # Suppress the 'ERROR:' prefix because this exception is not
                     # fatal and VS CI treat any logs with "ERROR:" as fatal.
                     mlog.exception(e, prefix=mlog.yellow('Exception:'))
-                mlog.log('\nSubproject', mlog.bold(subp_name), 'is buildable:', mlog.red('NO'), '(disabling)')
+                mlog.log('\nSubproject', mlog.bold(subdir), 'is buildable:', mlog.red('NO'), '(disabling)')
                 return self.disabled_subproject(subp_name, exception=e)
             raise e
 
@@ -2957,8 +2946,7 @@ external dependencies (including libraries) must go to "dependencies".''')
                 raise InterpreterException('Subproject %s version is %s but %s required.' % (subp_name, pv, wanted))
         self.active_projectname = current_active
         self.subprojects.update(subi.subprojects)
-        self.subprojects[subp_name] = SubprojectHolder(subi, self.subproject_dir, subp_name,
-                                                     warnings=subi_warnings)
+        self.subprojects[subp_name] = SubprojectHolder(subi, subdir, warnings=subi_warnings)
         # Duplicates are possible when subproject uses files from project root
         if build_def_files:
             self.build_def_files = list(set(self.build_def_files + build_def_files))
@@ -3157,8 +3145,11 @@ external dependencies (including libraries) must go to "dependencies".''')
                                               'license': proj_license}
         if self.subproject in self.build.projects:
             raise InvalidCode('Second call to project().')
-        if not self.is_subproject() and 'subproject_dir' in kwargs:
-            spdirname = kwargs['subproject_dir']
+
+        # spdirname is the subproject_dir for this project, relative to self.subdir.
+        # self.subproject_dir is the subproject_dir for the main project, relative to top source dir.
+        spdirname = kwargs.get('subproject_dir')
+        if spdirname:
             if not isinstance(spdirname, str):
                 raise InterpreterException('Subproject_dir must be a string')
             if os.path.isabs(spdirname):
@@ -3167,13 +3158,20 @@ external dependencies (including libraries) must go to "dependencies".''')
                 raise InterpreterException('Subproject_dir must not begin with a period.')
             if '..' in spdirname:
                 raise InterpreterException('Subproject_dir must not contain a ".." segment.')
-            self.subproject_dir = spdirname
-
+            if not self.is_subproject():
+                self.subproject_dir = spdirname
+        else:
+            spdirname = 'subprojects'
         self.build.subproject_dir = self.subproject_dir
-        if not self.is_subproject():
-            wrap_mode = self.coredata.get_builtin_option('wrap_mode')
-            subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
-            self.environment.wrap_resolver = wrap.Resolver(subproject_dir_abs, wrap_mode)
+
+        # Load wrap files from this (sub)project.
+        wrap_mode = self.coredata.get_builtin_option('wrap_mode')
+        subdir = os.path.join(self.subdir, spdirname)
+        r = wrap.Resolver(self.environment.get_source_dir(), subdir, wrap_mode)
+        if self.is_subproject():
+            self.environment.wrap_resolver.merge_wraps(r)
+        else:
+            self.environment.wrap_resolver = r
 
         self.build.projects[self.subproject] = proj_name
         mlog.log('Project name:', mlog.bold(proj_name))
@@ -3594,7 +3592,6 @@ external dependencies (including libraries) must go to "dependencies".''')
     def get_subproject_dep(self, name, display_name, subp_name, varname, kwargs):
         required = kwargs.get('required', True)
         wanted = mesonlib.stringlistify(kwargs.get('version', []))
-        subproj_path = os.path.join(self.subproject_dir, subp_name)
         dep = self.notfound_dependency()
         try:
             subproject = self.subprojects[subp_name]
@@ -3609,9 +3606,9 @@ external dependencies (including libraries) must go to "dependencies".''')
                 else:
                     if required:
                         m = 'Subproject {} did not override dependency {}'
-                        raise DependencyException(m.format(subproj_path, display_name))
+                        raise DependencyException(m.format(subproject.subdir, display_name))
                     mlog.log('Dependency', mlog.bold(display_name), 'from subproject',
-                             mlog.bold(subproj_path), 'found:', mlog.red('NO'))
+                             mlog.bold(subproject.subdir), 'found:', mlog.red('NO'))
                     return self.notfound_dependency()
             if subproject.found():
                 self.verify_fallback_consistency(subp_name, varname, cached_dep)
@@ -3629,7 +3626,7 @@ external dependencies (including libraries) must go to "dependencies".''')
                                           ''.format(varname, subp_name))
             # If the dependency is not required, don't raise an exception
             mlog.log('Dependency', mlog.bold(display_name), 'from subproject',
-                     mlog.bold(subproj_path), 'found:', mlog.red('NO'))
+                     mlog.bold(subproject.subdir), 'found:', mlog.red('NO'))
             return dep
 
         found = dep.held_object.get_version()
@@ -3640,14 +3637,14 @@ external dependencies (including libraries) must go to "dependencies".''')
                                           'dep {}'.format(found, subp_name, wanted, display_name))
 
             mlog.log('Dependency', mlog.bold(display_name), 'from subproject',
-                     mlog.bold(subproj_path), 'found:', mlog.red('NO'),
+                     mlog.bold(subproject.subdir), 'found:', mlog.red('NO'),
                      'found', mlog.normal_cyan(found), 'but need:',
                      mlog.bold(', '.join(["'{}'".format(e) for e in wanted])))
             return self.notfound_dependency()
 
         found = mlog.normal_cyan(found) if found else None
         mlog.log('Dependency', mlog.bold(display_name), 'from subproject',
-                 mlog.bold(subproj_path), 'found:', mlog.green('YES'), found)
+                 mlog.bold(subproject.subdir), 'found:', mlog.green('YES'), found)
         return dep
 
     def _handle_featurenew_dependencies(self, name):
@@ -3798,23 +3795,6 @@ external dependencies (including libraries) must go to "dependencies".''')
     def func_disabler(self, node, args, kwargs):
         return Disabler()
 
-    def print_nested_info(self, dependency_name):
-        message = ['Dependency', mlog.bold(dependency_name), 'not found but it is available in a sub-subproject.\n' +
-                   'To use it in the current project, promote it by going in the project source\n'
-                   'root and issuing']
-        sprojs = mesonlib.detect_subprojects('subprojects', self.source_root)
-        if dependency_name not in sprojs:
-            return
-        found = sprojs[dependency_name]
-        if len(found) > 1:
-            message.append('one of the following commands:')
-        else:
-            message.append('the following command:')
-        command_templ = '\nmeson wrap promote {}'
-        for l in found:
-            message.append(mlog.bold(command_templ.format(l[len(self.source_root) + 1:])))
-        mlog.warning(*message, location=self.current_node)
-
     def get_subproject_infos(self, fbinfo):
         fbinfo = mesonlib.stringlistify(fbinfo)
         if len(fbinfo) == 1:
@@ -4926,14 +4906,8 @@ This will become a hard error in the future.''', location=self.current_node)
 
     # Only permit object extraction from the same subproject
     def validate_extraction(self, buildtarget: InterpreterObject) -> None:
-        if not self.subdir.startswith(self.subproject_dir):
-            if buildtarget.subdir.startswith(self.subproject_dir):
-                raise InterpreterException('Tried to extract objects from a subproject target.')
-        else:
-            if not buildtarget.subdir.startswith(self.subproject_dir):
-                raise InterpreterException('Tried to extract objects from the main project from a subproject.')
-            if self.subdir.split('/')[1] != buildtarget.subdir.split('/')[1]:
-                raise InterpreterException('Tried to extract objects from a different subproject.')
+        if self.subproject != buildtarget.subproject:
+            raise InterpreterException('Tried to extract objects from a different subproject.')
 
     def is_subproject(self):
         return self.subproject != ''

mesonbuild/msubprojects.py

@@ -364,7 +364,7 @@ def run(options):
     if not os.path.isdir(subprojects_dir):
         mlog.log('Directory', mlog.bold(src_dir), 'does not seem to have subprojects.')
         return 0
-    r = Resolver(subprojects_dir)
+    r = Resolver(src_dir, 'subprojects')
     if options.subprojects:
         wraps = [wrap for name, wrap in r.wraps.items() if name in options.subprojects]
     else:

mesonbuild/wrap/wrap.py

@@ -97,11 +97,11 @@ class PackageDefinition:
         self.provided_deps = {} # type: T.Dict[str, T.Optional[str]]
         self.provided_programs = [] # type: T.List[str]
         self.basename = os.path.basename(fname)
-        self.name = self.basename
-        if self.name.endswith('.wrap'):
-            self.name = self.name[:-5]
+        self.has_wrap = self.basename.endswith('.wrap')
+        self.name = self.basename[:-5] if self.has_wrap else self.basename
+        self.directory = self.name
         self.provided_deps[self.name] = None
-        if fname.endswith('.wrap'):
+        if self.has_wrap:
             self.parse_wrap(fname)
         self.directory = self.values.get('directory', self.name)
         if os.path.dirname(self.directory):
@@ -164,9 +164,11 @@ def get_directory(subdir_root: str, packagename: str) -> str:
     return packagename
 
 class Resolver:
-    def __init__(self, subdir_root: str, wrap_mode: WrapMode = WrapMode.default) -> None:
+    def __init__(self, source_dir: str, subdir: str, wrap_mode: WrapMode = WrapMode.default) -> None:
+        self.source_dir = source_dir
+        self.subdir = subdir
         self.wrap_mode = wrap_mode
-        self.subdir_root = subdir_root
+        self.subdir_root = os.path.join(source_dir, subdir)
         self.cachedir = os.path.join(self.subdir_root, 'packagecache')
         self.filesdir = os.path.join(self.subdir_root, 'packagefiles')
         self.wraps = {} # type: T.Dict[str, PackageDefinition]
@@ -208,6 +210,14 @@ class Resolver:
                     raise WrapException(m.format(k, wrap.basename, prev_wrap.basename))
                 self.provided_programs[k] = wrap
 
+    def merge_wraps(self, other_resolver: 'Resolver') -> None:
+        for k, v in other_resolver.wraps.items():
+            self.wraps.setdefault(k, v)
+        for k, v in other_resolver.provided_deps.items():
+            self.provided_deps.setdefault(k, v)
+        for k, v in other_resolver.provided_programs.items():
+            self.provided_programs.setdefault(k, v)
+
     def find_dep_provider(self, packagename: str) -> T.Optional[T.Union[str, T.List[str]]]:
         # Return value is in the same format as fallback kwarg:
         # ['subproject_name', 'variable_name'], or 'subproject_name'.
@@ -235,7 +245,24 @@ class Resolver:
             m = 'Subproject directory not found and {}.wrap file not found'
             raise WrapNotFoundException(m.format(self.packagename))
         self.directory = self.wrap.directory
-        self.dirname = os.path.join(self.subdir_root, self.directory)
+
+        if self.wrap.has_wrap:
+            # We have a .wrap file, source code will be placed into main
+            # project's subproject_dir even if the wrap file comes from another
+            # subproject.
+            self.dirname = os.path.join(self.subdir_root, self.directory)
+            # Copy .wrap file into main project's subproject_dir
+            wrap_dir = os.path.normpath(os.path.dirname(self.wrap.filename))
+            main_dir = os.path.normpath(self.subdir_root)
+            if wrap_dir != main_dir:
+                rel = os.path.relpath(self.wrap.filename, self.source_dir)
+                mlog.log('Using', mlog.bold(rel))
+                shutil.copy2(self.wrap.filename, self.subdir_root)
+        else:
+            # No wrap file, it's a dummy package definition for an existing
+            # directory. Use the source code in place.
+            self.dirname = self.wrap.filename
+        rel_path = os.path.relpath(self.dirname, self.source_dir)
 
         meson_file = os.path.join(self.dirname, 'meson.build')
         cmake_file = os.path.join(self.dirname, 'CMakeLists.txt')
@@ -245,9 +272,9 @@ class Resolver:
 
         # The directory is there and has meson.build? Great, use it.
         if method == 'meson' and os.path.exists(meson_file):
-            return self.directory
+            return rel_path
         if method == 'cmake' and os.path.exists(cmake_file):
-            return self.directory
+            return rel_path
 
         # Check if the subproject is a git submodule
         self.resolve_git_submodule()
@@ -276,7 +303,7 @@ class Resolver:
         if method == 'cmake' and not os.path.exists(cmake_file):
             raise WrapException('Subproject exists but has no CMakeLists.txt file')
 
-        return self.directory
+        return rel_path
 
     def check_can_download(self) -> None:
         # Don't download subproject data based on wrap file if requested.

run_unittests.py

@@ -4183,6 +4183,16 @@ recommended as it is not supported on some platforms''')
                     'name': 'sub_novar',
                     'version': '1.0',
                 },
+                {
+                    'descriptive_name': 'subsub',
+                    'name': 'subsub',
+                    'version': 'undefined'
+                },
+                {
+                    'descriptive_name': 'subsubsub',
+                    'name': 'subsubsub',
+                    'version': 'undefined'
+                },
             ]
         }
         res['subprojects'] = sorted(res['subprojects'], key=lambda i: i['name'])
@@ -5365,16 +5375,16 @@ class FailureTests(BasePlatformTests):
            correct message when the fallback subproject is found but the
            variable inside it is not.
         4. A fallback dependency is found from the subproject parsed in (3)
-        5. The correct message is outputted when the .wrap file is missing for
-           a sub-subproject.
+        5. A wrap file from a subproject is used but fails because it does not
+           contain required keys.
         '''
         tdir = os.path.join(self.unit_test_dir, '20 subproj dep variables')
         out = self.init(tdir, inprocess=True)
         self.assertRegex(out, r"Subproject directory not found and .*nosubproj.wrap.* file not found")
         self.assertRegex(out, r'Function does not take positional arguments.')
-        self.assertRegex(out, r'WARNING:.* Dependency .*subsubproject.* not found but it is available in a sub-subproject.')
-        self.assertRegex(out, r'Subproject directory not found and .*subsubproject.wrap.* file not found')
+        self.assertRegex(out, r'Dependency .*somenotfounddep.* from subproject .*subprojects/somesubproj.* found: .*NO.*')
         self.assertRegex(out, r'Dependency .*zlibproxy.* from subproject .*subprojects.*somesubproj.* found: .*YES.*')
+        self.assertRegex(out, r'Missing key .*source_filename.* in subsubproject.wrap')
 
     def test_exception_exit_status(self):
         '''

test cases/common/102 subproject subdir/meson.build

@@ -54,3 +54,14 @@ assert(d.type_name() == 'internal')
 # Using gobject-2.0 here because some CI runners have it installed.
 d = dependency('gobject-2.0', required : false)
 assert(not d.found())
+
+# Verify that implicit fallback works because subprojects/sub_implicit/subprojects/subsub
+# directory exists.
+d = dependency('subsub')
+assert(d.found(), 'Should be able to fallback to sub-subproject')
+
+# Verify that implicit fallback works because
+# subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap
+# file exists.
+d = dependency('subsubsub')
+assert(d.found(), 'Should be able to fallback to sub-sub-subproject')

test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/meson.build

@@ -0,0 +1,3 @@
+project('subsub')
+
+meson.override_dependency('subsub', declare_dependency())

test cases/common/102 subproject subdir/subprojects/sub_implicit/subprojects/subsub/subprojects/subsubsub.wrap

@@ -0,0 +1,4 @@
+[wrap-file]
+directory = subsubsub-1.0
+source_filename = subsubsub-1.0.zip
+source_hash = c073a96b7251937e53216578f6f03d91b84816618a0f1ce3ecfb867beddf1498

test cases/failing/16 extract from subproject/test.json

@@ -1,7 +1,7 @@
 {
   "stdout": [
     {
-      "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a subproject target."
+      "line": "test cases/failing/16 extract from subproject/meson.build:6:0: ERROR: Tried to extract objects from a different subproject."
     }
   ]
 }