Merge pull request #8941 from xclaesse/wrapdb

Automatically use WrapDB fallback

docs/markdown/Using-wraptool.md

@@ -82,3 +82,17 @@ straightforward:
 Wraptool can do other things besides these. Documentation for these
 can be found in the command line help, which can be accessed by
 `meson wrap --help`.
+
+## Automatic dependency fallback
+
+Since *0.64.0* Meson can use WrapDB to automatically find missing dependencies.
+
+The user simply needs to download latest database, the following command stores
+it in `subprojects/wrapdb.json`:
+    $ meson wrap update-db
+
+Once the database is available locally, any dependency not found on the system
+but available in WrapDB will automatically be downloaded.
+
+Automatic fetch of WrapDB subprojects can be disabled by removing the file
+`subprojects/wrapdb.json`, or by using `--wrap-mode=nodownload`.

docs/markdown/snippets/wrapdb.md

@@ -0,0 +1,6 @@
+## Automatic fallback using WrapDB
+
+A new command has been added: `meson wrap update-db`. It downloads the list of
+wraps available in [WrapDB](wrapdb.mesonbuild.com) and stores it locally in
+`subprojects/wrapdb.json`. When that file exists and a dependency is not found
+on the system but is available in WrapDB, Meson will automatically download it.

docs/yaml/functions/dependency.yaml

@@ -15,6 +15,11 @@ description: |
   of those name will return the same value. This is useful in case a dependency
   could have different names, such as `png` and `libpng`.
 
+  * Since *0.64.0* a dependency fallback can be provided by WrapDB. Simply download
+  the database locally using `meson wrap update-db` command and Meson will
+  automatically fallback to subprojects provided by WrapDB if the dependency is
+  not found on the system and the project does not ship their own `.wrap` file.
+
   Dependencies can also be resolved in two other ways:
 
   * if the same name was used in a `meson.override_dependency` prior to

mesonbuild/interpreter/dependencyfallbacks.py

@@ -66,14 +66,6 @@ class DependencyFallbacksHolder(MesonInterpreterObject):
         self._subproject_impl(subp_name, varname)
 
     def _subproject_impl(self, subp_name: str, varname: str) -> None:
-        if not varname:
-            # If no variable name is specified, check if the wrap file has one.
-            # If the wrap file has a variable name, better use it because the
-            # subproject most probably is not using meson.override_dependency().
-            for name in self.names:
-                varname = self.wrap_resolver.get_varname(subp_name, name)
-                if varname:
-                    break
         assert self.subproject_name is None
         self.subproject_name = subp_name
         self.subproject_varname = varname
@@ -174,6 +166,14 @@ class DependencyFallbacksHolder(MesonInterpreterObject):
 
         # Legacy: Use the variable name if provided instead of relying on the
         # subproject to override one of our dependency names
+        if not varname:
+            # If no variable name is specified, check if the wrap file has one.
+            # If the wrap file has a variable name, better use it because the
+            # subproject most probably is not using meson.override_dependency().
+            for name in self.names:
+                varname = self.wrap_resolver.get_varname(subp_name, name)
+                if varname:
+                    break
         if not varname:
             mlog.warning(f'Subproject {subp_name!r} did not override {self._display_name!r} dependency and no variable name specified')
             mlog.log('Dependency', mlog.bold(self._display_name), 'from subproject',

mesonbuild/wrap/wrap.py

@@ -29,10 +29,12 @@ import configparser
 import time
 import typing as T
 import textwrap
+import json
 
 from base64 import b64encode
 from netrc import netrc
 from pathlib import Path
+
 from . import WrapMode
 from .. import coredata
 from ..mesonlib import quiet_git, GIT, ProgressBar, MesonException, windows_proof_rmtree, Popen_safe
@@ -262,8 +264,12 @@ class Resolver:
         self.netrc: T.Optional[netrc] = None
         self.provided_deps = {} # type: T.Dict[str, PackageDefinition]
         self.provided_programs = {} # type: T.Dict[str, PackageDefinition]
+        self.wrapdb: T.Dict[str, T.Any] = {}
+        self.wrapdb_provided_deps: T.Dict[str, str] = {}
+        self.wrapdb_provided_programs: T.Dict[str, str] = {}
         self.load_wraps()
         self.load_netrc()
+        self.load_wrapdb()
 
     def load_netrc(self) -> None:
         try:
@@ -294,18 +300,48 @@ class Resolver:
             self.wraps[wrap.name] = wrap
 
         for wrap in self.wraps.values():
-            for k in wrap.provided_deps.keys():
-                if k in self.provided_deps:
-                    prev_wrap = self.provided_deps[k]
-                    m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}'
-                    raise WrapException(m)
-                self.provided_deps[k] = wrap
-            for k in wrap.provided_programs:
-                if k in self.provided_programs:
-                    prev_wrap = self.provided_programs[k]
-                    m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}'
-                    raise WrapException(m)
-                self.provided_programs[k] = wrap
+            self.add_wrap(wrap)
+
+    def add_wrap(self, wrap: PackageDefinition) -> None:
+        for k in wrap.provided_deps.keys():
+            if k in self.provided_deps:
+                prev_wrap = self.provided_deps[k]
+                m = f'Multiple wrap files provide {k!r} dependency: {wrap.basename} and {prev_wrap.basename}'
+                raise WrapException(m)
+            self.provided_deps[k] = wrap
+        for k in wrap.provided_programs:
+            if k in self.provided_programs:
+                prev_wrap = self.provided_programs[k]
+                m = f'Multiple wrap files provide {k!r} program: {wrap.basename} and {prev_wrap.basename}'
+                raise WrapException(m)
+            self.provided_programs[k] = wrap
+
+    def load_wrapdb(self) -> None:
+        try:
+            with Path(self.subdir_root, 'wrapdb.json').open('r', encoding='utf-8') as f:
+                self.wrapdb = json.load(f)
+        except FileNotFoundError:
+            return
+        for name, info in self.wrapdb.items():
+            self.wrapdb_provided_deps.update({i: name for i in info.get('dependency_names', [])})
+            self.wrapdb_provided_programs.update({i: name for i in info.get('program_names', [])})
+
+    def get_from_wrapdb(self, subp_name: str) -> PackageDefinition:
+        info = self.wrapdb.get(subp_name)
+        if not info:
+            return None
+        self.check_can_download()
+        latest_version = info['versions'][0]
+        version, revision = latest_version.rsplit('-', 1)
+        url = urllib.request.urlopen(f'https://wrapdb.mesonbuild.com/v2/{subp_name}_{version}-{revision}/{subp_name}.wrap')
+        fname = Path(self.subdir_root, f'{subp_name}.wrap')
+        with fname.open('wb') as f:
+            f.write(url.read())
+        mlog.log(f'Installed {subp_name} version {version} revision {revision}')
+        wrap = PackageDefinition(str(fname))
+        self.wraps[wrap.name] = wrap
+        self.add_wrap(wrap)
+        return wrap
 
     def merge_wraps(self, other_resolver: 'Resolver') -> None:
         for k, v in other_resolver.wraps.items():
@@ -323,7 +359,8 @@ class Resolver:
         if wrap:
             dep_var = wrap.provided_deps.get(packagename)
             return wrap.name, dep_var
-        return None, None
+        wrap_name = self.wrapdb_provided_deps.get(packagename)
+        return wrap_name, None
 
     def get_varname(self, subp_name: str, depname: str) -> T.Optional[str]:
         wrap = self.wraps.get(subp_name)
@@ -334,12 +371,17 @@ class Resolver:
             wrap = self.provided_programs.get(name)
             if wrap:
                 return wrap.name
+            wrap_name = self.wrapdb_provided_programs.get(name)
+            if wrap_name:
+                return wrap_name
         return None
 
     def resolve(self, packagename: str, method: str) -> str:
         self.packagename = packagename
         self.directory = packagename
         self.wrap = self.wraps.get(packagename)
+        if not self.wrap:
+            self.wrap = self.get_from_wrapdb(packagename)
         if not self.wrap:
             m = f'Neither a subproject directory nor a {self.packagename}.wrap file was found.'
             raise WrapNotFoundException(m)

mesonbuild/wrap/wraptool.py

@@ -21,6 +21,7 @@ import typing as T
 from glob import glob
 from urllib.parse import urlparse
 from .wrap import open_wrapdburl, WrapException
+from pathlib import Path
 
 from .. import mesonlib
 
@@ -69,9 +70,18 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('project_path')
     p.set_defaults(wrap_func=promote)
 
-def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    p = subparsers.add_parser('update-db', help='Update list of projects available in WrapDB (Since 0.61.0)')
+    p.add_argument('--allow-insecure', default=False, action='store_true',
+                   help='Allow insecure server connections.')
+    p.set_defaults(wrap_func=update_db)
+
+def get_releases_data(allow_insecure: bool) -> bytes:
     url = open_wrapdburl('https://wrapdb.mesonbuild.com/v2/releases.json', allow_insecure, True)
-    return T.cast('T.Dict[str, T.Any]', json.loads(url.read().decode()))
+    return url.read()
+
+def get_releases(allow_insecure: bool) -> T.Dict[str, T.Any]:
+    data = get_releases_data(allow_insecure)
+    return T.cast('T.Dict[str, T.Any]', json.loads(data.decode()))
 
 def list_projects(options: 'argparse.Namespace') -> None:
     releases = get_releases(options.allow_insecure)
@@ -244,6 +254,12 @@ def status(options: 'argparse.Namespace') -> None:
         else:
             print('', name, f'not up to date. Have {current_branch} {current_revision}, but {latest_branch} {latest_revision} is available.')
 
+def update_db(options: 'argparse.Namespace') -> None:
+    data = get_releases_data(options.allow_insecure)
+    Path('subprojects').mkdir(exist_ok=True)
+    with Path('subprojects/wrapdb.json').open('wb') as f:
+        f.write(data)
+
 def run(options: 'argparse.Namespace') -> int:
     options.wrap_func(options)
     return 0

unittests/platformagnostictests.py

@@ -14,7 +14,10 @@
 
 import os
 import tempfile
+import subprocess
+import textwrap
 from unittest import skipIf
+from pathlib import Path
 
 from .baseplatformtests import BasePlatformTests
 from .helpers import is_ci
@@ -94,3 +97,19 @@ class PlatformAgnosticTests(BasePlatformTests):
         # https://github.com/mesonbuild/meson/issues/10225.
         self.setconf('-Dfoo=enabled')
         self.build('reconfigure')
+
+    def test_update_wrapdb(self):
+        # Write the project into a temporary directory because it will add files
+        # into subprojects/ and we don't want to pollute meson source tree.
+        with tempfile.TemporaryDirectory() as testdir:
+            with Path(testdir, 'meson.build').open('w', encoding='utf-8') as f:
+                f.write(textwrap.dedent(
+                    '''
+                    project('wrap update-db',
+                      default_options: ['wrap_mode=forcefallback'])
+
+                    zlib_dep = dependency('zlib')
+                    assert(zlib_dep.type_name() == 'internal')
+                    '''))
+            subprocess.check_call(self.wrap_command + ['update-db'], cwd=testdir)
+            self.init(testdir, workdir=testdir)