Merge pull request #8796 from xclaesse/wrapdbv2

wrap: Port to v2 protocol

docs/markdown/Wrapdb-projects.md

@@ -0,0 +1,16 @@
+# Meson WrapDB packages
+
+This is a list of projects that have either an upstream Meson build system, or a
+port maintained by the Meson team. [They can be used by your project to provide
+its dependencies](Wrap-dependency-system-manual.md).
+
+Use the command line `meson wrap install <project>` to install the wrap file of
+any of those projects into your project's `subprojects/` directory.
+See [Meson command line documentation](Using-wraptool.md).
+
+If you wish to add your own project into this list, please submit your wrap file
+in a [Pull Request](https://github.com/mesonbuild/wrapdb).
+See [Meson documentation](Adding-new-projects-to-wrapdb.md)
+for more details.
+
+{{ wrapdb-table.md }}

docs/sitemap.txt

@@ -75,6 +75,7 @@ index.md
 	FAQ.md
 	Reproducible-builds.md
 	howtox.md
+	Wrapdb-projects.md
 	Wrap-dependency-system-manual.md
 		Adding-new-projects-to-wrapdb.md
 		Using-the-WrapDB.md

mesonbuild/msubprojects.py

@@ -4,23 +4,24 @@ from pathlib import Path
 
 from . import mlog
 from .mesonlib import quiet_git, verbose_git, GitException, Popen_safe, MesonException, windows_proof_rmtree
-from .wrap.wrap import API_ROOT, PackageDefinition, Resolver, WrapException, ALL_TYPES
+from .wrap.wrap import PackageDefinition, Resolver, WrapException, ALL_TYPES
 from .wrap import wraptool
 
 ALL_TYPES_STRING = ', '.join(ALL_TYPES)
 
 def update_wrapdb_file(wrap):
-    patch_url = wrap.get('patch_url')
-    branch, revision = wraptool.parse_patch_url(patch_url)
+    try:
+        patch_url = wrap.get('patch_url')
+        branch, revision = wraptool.parse_patch_url(patch_url)
+    except WrapException:
+        return
     new_branch, new_revision = wraptool.get_latest_version(wrap.name)
     if new_branch != branch or new_revision != revision:
         wraptool.update_wrap_file(wrap.filename, wrap.name, new_branch, new_revision)
         mlog.log('  -> New wrap file downloaded.')
 
 def update_file(r, wrap, repo_dir, options):
-    patch_url = wrap.values.get('patch_url', '')
-    if patch_url.startswith(API_ROOT):
-        update_wrapdb_file(wrap)
+    update_wrapdb_file(wrap)
     if not os.path.isdir(repo_dir):
         # The subproject is not needed, or it is a tarball extracted in
         # 'libfoo-1.0' directory and the version has been bumped and the new

mesonbuild/wrap/wrap.py

@@ -42,10 +42,8 @@ try:
     # regarding 'imported but unused' can be safely ignored
     import ssl  # noqa
     has_ssl = True
-    API_ROOT = 'https://wrapdb.mesonbuild.com/v1/'
 except ImportError:
     has_ssl = False
-    API_ROOT = 'http://wrapdb.mesonbuild.com/v1/'
 
 REQ_TIMEOUT = 600.0
 SSL_WARNING_PRINTED = False

mesonbuild/wrap/wraptool.py

@@ -19,8 +19,9 @@ import shutil
 import typing as T
 
 from glob import glob
-
-from .wrap import API_ROOT, open_wrapdburl
+from urllib.parse import urlparse
+from urllib.request import urlopen
+from .wrap import WrapException
 
 from .. import mesonlib
 
@@ -57,38 +58,30 @@ def add_arguments(parser: 'argparse.ArgumentParser') -> None:
     p.add_argument('project_path')
     p.set_defaults(wrap_func=promote)
 
-def get_result(urlstring: str) -> T.Dict[str, T.Any]:
-    u = open_wrapdburl(urlstring)
-    data = u.read().decode('utf-8')
-    jd = json.loads(data)
-    if jd['output'] != 'ok':
-        print('Got bad output from server.', file=sys.stderr)
-        raise SystemExit(data)
-    assert isinstance(jd, dict)
-    return jd
-
-def get_projectlist() -> T.List[str]:
-    jd = get_result(API_ROOT + 'projects')
-    projects = jd['projects']
-    assert isinstance(projects, list)
-    return projects
+def get_releases() -> T.Dict[str, T.Any]:
+    url = urlopen('https://wrapdb.mesonbuild.com/v2/releases.json')
+    return T.cast(T.Dict[str, T.Any], json.loads(url.read().decode()))
 
 def list_projects(options: 'argparse.Namespace') -> None:
-    projects = get_projectlist()
-    for p in projects:
+    releases = get_releases()
+    for p in releases.keys():
         print(p)
 
 def search(options: 'argparse.Namespace') -> None:
     name = options.name
-    jd = get_result(API_ROOT + 'query/byname/' + name)
-    for p in jd['projects']:
-        print(p)
+    releases = get_releases()
+    for p in releases.keys():
+        if p.startswith(name):
+            print(p)
 
 def get_latest_version(name: str) -> tuple:
-    jd = get_result(API_ROOT + 'query/get_latest/' + name)
-    branch = jd['branch']
-    revision = jd['revision']
-    return branch, revision
+    releases = get_releases()
+    info = releases.get(name)
+    if not info:
+        raise WrapException(f'Wrap {name} not found in wrapdb')
+    latest_version = info['versions'][0]
+    version, revision = latest_version.rsplit('-', 1)
+    return version, revision
 
 def install(options: 'argparse.Namespace') -> None:
     name = options.name
@@ -99,16 +92,28 @@ def install(options: 'argparse.Namespace') -> None:
     wrapfile = os.path.join('subprojects', name + '.wrap')
     if os.path.exists(wrapfile):
         raise SystemExit('Wrap file already exists.')
-    (branch, revision) = get_latest_version(name)
-    u = open_wrapdburl(API_ROOT + f'projects/{name}/{branch}/{revision}/get_wrap')
-    data = u.read()
+    (version, revision) = get_latest_version(name)
+    url = urlopen(f'https://wrapdb.mesonbuild.com/v2/{name}_{version}-{revision}/{name}.wrap')
     with open(wrapfile, 'wb') as f:
-        f.write(data)
-    print('Installed', name, 'branch', branch, 'revision', revision)
+        f.write(url.read())
+    print(f'Installed {name} version {version} revision {revision}')
 
 def parse_patch_url(patch_url: str) -> T.Tuple[str, int]:
-    arr = patch_url.split('/')
-    return arr[-3], int(arr[-2])
+    u = urlparse(patch_url)
+    if u.netloc != 'wrapdb.mesonbuild.com':
+        raise WrapException(f'URL {patch_url} does not seems to be a wrapdb patch')
+    arr = u.path.strip('/').split('/')
+    if arr[0] == 'v1':
+        # e.g. https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/5/get_zip
+        return arr[-3], int(arr[-2])
+    elif arr[0] == 'v2':
+        # e.g. https://wrapdb.mesonbuild.com/v2/zlib_1.2.11-5/get_patch
+        tag = arr[-2]
+        name, version = tag.rsplit('_', 1)
+        version, revision = version.rsplit('-', 1)
+        return version, int(revision)
+    else:
+        raise WrapException(f'Invalid wrapdb URL {patch_url}')
 
 def get_current_version(wrapfile: str) -> T.Tuple[str, int, str, str, str]:
     cp = configparser.ConfigParser(interpolation=None)
@@ -118,11 +123,10 @@ def get_current_version(wrapfile: str) -> T.Tuple[str, int, str, str, str]:
     branch, revision = parse_patch_url(patch_url)
     return branch, revision, wrap_data['directory'], wrap_data['source_filename'], wrap_data['patch_filename']
 
-def update_wrap_file(wrapfile: str, name: str, new_branch: str, new_revision: str) -> None:
-    u = open_wrapdburl(API_ROOT + f'projects/{name}/{new_branch}/{new_revision}/get_wrap')
-    data = u.read()
+def update_wrap_file(wrapfile: str, name: str, new_version: str, new_revision: str) -> None:
+    url = urlopen(f'https://wrapdb.mesonbuild.com/v2/{name}_{new_version}-{new_revision}/{name}.wrap')
     with open(wrapfile, 'wb') as f:
-        f.write(data)
+        f.write(url.read())
 
 def update(options: 'argparse.Namespace') -> None:
     name = options.name
@@ -146,17 +150,17 @@ def update(options: 'argparse.Namespace') -> None:
         os.unlink(os.path.join('subprojects/packagecache', patch_file))
     except FileNotFoundError:
         pass
-    print('Updated', name, 'to branch', new_branch, 'revision', new_revision)
+    print(f'Updated {name} version {new_branch} revision {new_revision}')
 
 def info(options: 'argparse.Namespace') -> None:
     name = options.name
-    jd = get_result(API_ROOT + 'projects/' + name)
-    versions = jd['versions']
-    if not versions:
-        raise SystemExit('No available versions of' + name)
+    releases = get_releases()
+    info = releases.get(name)
+    if not info:
+        raise WrapException(f'Wrap {name} not found in wrapdb')
     print(f'Available versions of {name}:')
-    for v in versions:
-        print(' ', v['branch'], v['revision'])
+    for v in info['versions']:
+        print(' ', v)
 
 def do_promotion(from_path: str, spdir_name: str) -> None:
     if os.path.isfile(from_path):

tools/regenerate_docs.py

@@ -25,8 +25,10 @@ import re
 import subprocess
 import sys
 import textwrap
+import json
 import typing as T
 from pathlib import Path
+from urllib.request import urlopen
 
 PathLike = T.Union[Path,str]
 
@@ -115,6 +117,22 @@ def generate_hotdoc_includes(root_dir: Path, output_dir: Path) -> None:
             with open(output_dir / (cmd+'_'+typ+'.inc'), 'w') as f:
                 f.write(parsed[typ])
 
+def generate_wrapdb_table(output_dir: Path) -> None:
+    url = urlopen('https://wrapdb.mesonbuild.com/v2/releases.json')
+    releases = json.loads(url.read().decode())
+    with open(output_dir / 'wrapdb-table.md', 'w') as f:
+        f.write('| Project | Versions | Provided dependencies | Provided programs |\n')
+        f.write('| ------- | -------- | --------------------- | ----------------- |\n')
+        for name, info in releases.items():
+            versions = [f'[{v}](https://wrapdb.mesonbuild.com/v2/{name}_{v}/{name}.wrap)' for v in info['versions']]
+            # Highlight latest version.
+            versions_str = f'<big>**{versions[0]}**</big><br/>' + ', '.join(versions[1:])
+            dependency_names = info.get('dependency_names', [])
+            dependency_names_str = ', '.join(dependency_names)
+            program_names = info.get('program_names', [])
+            program_names_str = ', '.join(program_names)
+            f.write(f'| {name} | {versions_str} | {dependency_names_str} | {program_names_str} |\n')
+
 def regenerate_docs(output_dir: PathLike,
                     dummy_output_file: T.Optional[PathLike]) -> None:
     if not output_dir:
@@ -126,6 +144,7 @@ def regenerate_docs(output_dir: PathLike,
     root_dir = Path(__file__).resolve().parent.parent
 
     generate_hotdoc_includes(root_dir, output_dir)
+    generate_wrapdb_table(output_dir)
 
     if dummy_output_file:
         with open(output_dir/dummy_output_file, 'w') as f: