Add 'meson subprojects update' command

This is inspired by gst-build's git-update script.

mesonbuild/mesonmain.py

@@ -20,7 +20,7 @@ import argparse
 
 from . import mesonlib
 from . import mlog
-from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter
+from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects
 from .mesonlib import MesonException
 from .environment import detect_msys2_arch
 from .wrap import wraptool
@@ -47,6 +47,8 @@ class CommandLineParser:
                          help='Run tests')
         self.add_command('wrap', wraptool.add_arguments, wraptool.run,
                          help='Wrap tools')
+        self.add_command('subprojects', msubprojects.add_arguments, msubprojects.run,
+                         help='Manage subprojects')
         self.add_command('help', self.add_help_arguments, self.run_help_command,
                          help='Print help of a subcommand')
 

mesonbuild/mlog.py

@@ -96,6 +96,9 @@ def green(text):
 def yellow(text):
     return AnsiDecorator(text, "\033[1;33m")
 
+def blue(text):
+    return AnsiDecorator(text, "\033[1;34m")
+
 def cyan(text):
     return AnsiDecorator(text, "\033[1;36m")
 

mesonbuild/msubprojects.py

@@ -0,0 +1,179 @@
+import os, subprocess
+
+from . import mlog
+from .mesonlib import Popen_safe
+from .wrap.wrap import API_ROOT, PackageDefinition
+from .wrap import wraptool
+
+def update_wrapdb_file(wrap, repo_dir, options):
+    patch_url = wrap.get('patch_url')
+    branch, revision = wraptool.parse_patch_url(patch_url)
+    new_branch, new_revision = wraptool.get_latest_version(wrap.name)
+    if new_branch == branch and new_revision == revision:
+        mlog.log('  -> Up to date.')
+        return
+    wraptool.update_wrap_file(wrap.filename, wrap.name, new_branch, new_revision)
+    msg = ['  -> New wrap file downloaded.']
+    # Meson reconfigure won't use the new wrap file as long as the source
+    # directory exists. We don't delete it ourself to avoid data loss in case
+    # user has changes in their copy.
+    if os.path.isdir(repo_dir):
+        msg += ['To use it, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure')]
+    mlog.log(*msg)
+
+def update_file(wrap, repo_dir, options):
+    patch_url = wrap.values.get('patch_url', '')
+    if patch_url.startswith(API_ROOT):
+        update_wrapdb_file(wrap, repo_dir, options)
+    elif not os.path.isdir(repo_dir):
+        # The subproject is not needed, or it is a tarball extracted in
+        # 'libfoo-1.0' directory and the version has been bumped and the new
+        # directory is 'libfoo-2.0'. In that case forcing a meson
+        # reconfigure will download and use the new tarball.
+        mlog.log('  -> Subproject has not been checked out. Run', mlog.bold('meson --reconfigure'), 'to fetch it if needed.')
+    else:
+        # The subproject has not changed, or the new source and/or patch
+        # tarballs should be extracted in the same directory than previous
+        # version.
+        mlog.log('  -> Subproject has not changed, or the new source/patch needs to be extracted on the same location.\n' +
+                 '     In that case, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure'))
+
+def git(cmd, workingdir):
+    return subprocess.check_output(['git', '-C', workingdir] + cmd,
+                                   stderr=subprocess.STDOUT).decode()
+
+def update_git(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revision = wrap.get('revision')
+    ret = git(['rev-parse', '--abbrev-ref', 'HEAD'], repo_dir).strip()
+    if ret == 'HEAD':
+        try:
+            # We are currently in detached mode, just checkout the new revision
+            git(['fetch'], repo_dir)
+            git(['checkout', revision], repo_dir)
+        except subprocess.CalledProcessError as e:
+            out = e.output.decode().strip()
+            mlog.log('  -> Could not checkout revision', mlog.cyan(revision))
+            mlog.log(mlog.red(out))
+            mlog.log(mlog.red(str(e)))
+            return
+    elif ret == revision:
+        try:
+            # We are in the same branch, pull latest commits
+            git(['-c', 'rebase.autoStash=true', 'pull', '--rebase'], repo_dir)
+        except subprocess.CalledProcessError as e:
+            out = e.output.decode().strip()
+            mlog.log('  -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.')
+            mlog.log(mlog.red(out))
+            mlog.log(mlog.red(str(e)))
+            return
+    else:
+        # We are in another branch, probably user created their own branch and
+        # we should rebase it on top of wrap's branch.
+        if options.rebase:
+            try:
+                git(['fetch'], repo_dir)
+                git(['-c', 'rebase.autoStash=true', 'rebase', revision], repo_dir)
+            except subprocess.CalledProcessError as e:
+                out = e.output.decode().strip()
+                mlog.log('  -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.')
+                mlog.log(mlog.red(out))
+                mlog.log(mlog.red(str(e)))
+                return
+        else:
+            mlog.log('  -> Target revision is', mlog.bold(revision), 'but currently in branch is', mlog.bold(ret), '\n' +
+                     '     To rebase your branch on top of', mlog.bold(revision), 'use', mlog.bold('--rebase'), 'option.')
+            return
+
+    git(['submodule', 'update'], repo_dir)
+    commit_message = git(['show', '--quiet', '--pretty=format:%h%n%d%n%s%n[%an]'], repo_dir)
+    parts = [s.strip() for s in commit_message.split('\n')]
+    mlog.log('  ->', mlog.yellow(parts[0]), mlog.red(parts[1]), parts[2], mlog.blue(parts[3]))
+
+def update_hg(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revno = wrap.get('revision')
+    if revno.lower() == 'tip':
+        # Failure to do pull is not a fatal error,
+        # because otherwise you can't develop without
+        # a working net connection.
+        subprocess.call(['hg', 'pull'], cwd=repo_dir)
+    else:
+        if subprocess.call(['hg', 'checkout', revno], cwd=repo_dir) != 0:
+            subprocess.check_call(['hg', 'pull'], cwd=repo_dir)
+            subprocess.check_call(['hg', 'checkout', revno], cwd=repo_dir)
+
+def update_svn(wrap, repo_dir, options):
+    if not os.path.isdir(repo_dir):
+        mlog.log('  -> Not used.')
+        return
+    revno = wrap.get('revision')
+    p, out = Popen_safe(['svn', 'info', '--show-item', 'revision', repo_dir])
+    current_revno = out
+    if current_revno == revno:
+        return
+    if revno.lower() == 'head':
+        # Failure to do pull is not a fatal error,
+        # because otherwise you can't develop without
+        # a working net connection.
+        subprocess.call(['svn', 'update'], cwd=repo_dir)
+    else:
+        subprocess.check_call(['svn', 'update', '-r', revno], cwd=repo_dir)
+
+def update(options):
+    src_dir = os.path.relpath(os.path.realpath(options.sourcedir))
+    if not os.path.isfile(os.path.join(src_dir, 'meson.build')):
+        mlog.error('Directory', mlog.bold(src_dir), 'does not seem to be a Meson source directory.')
+        return 1
+    subprojects_dir = os.path.join(src_dir, 'subprojects')
+    if not os.path.isdir(subprojects_dir):
+        mlog.log('Directory', mlog.bold(src_dir), 'does not seem to have subprojects.')
+        return 0
+    files = []
+    for name in options.subprojects:
+        f = os.path.join(subprojects_dir, name + '.wrap')
+        if not os.path.isfile(f):
+            mlog.error('Subproject', mlog.bold(name), 'not found.')
+            return 1
+        else:
+            files.append(f)
+    if not files:
+        for f in os.listdir(subprojects_dir):
+            if f.endswith('.wrap'):
+                files.append(os.path.join(subprojects_dir, f))
+    for f in files:
+        wrap = PackageDefinition(f)
+        directory = wrap.values.get('directory', wrap.name)
+        dirname = os.path.join(subprojects_dir, directory)
+        mlog.log('Updating %s...' % wrap.name)
+        if wrap.type == 'file':
+            update_file(wrap, dirname, options)
+        elif wrap.type == 'git':
+            update_git(wrap, dirname, options)
+        elif wrap.type == 'hg':
+            update_hg(wrap, dirname, options)
+        elif wrap.type == 'svn':
+            update_svn(wrap, dirname, options)
+        else:
+            mlog.log('  -> Cannot update', wrap.type, 'subproject')
+    return 0
+
+def add_arguments(parser):
+    subparsers = parser.add_subparsers(title='Commands', dest='command')
+    subparsers.required = True
+
+    p = subparsers.add_parser('update', help='Update all subprojects from wrap files')
+    p.add_argument('--sourcedir', default='.',
+                   help='Path to source directory')
+    p.add_argument('--rebase', default=False, action='store_true',
+                   help='Rebase your branch on top of wrap\'s revision (git only)')
+    p.add_argument('subprojects', nargs='*',
+                   help='List of subprojects (default: all)')
+    p.set_defaults(subprojects_func=update)
+
+def run(options):
+    return options.subprojects_func(options)

mesonbuild/wrap/wrap.py

@@ -78,7 +78,9 @@ class WrapNotFoundException(WrapException):
 
 class PackageDefinition:
     def __init__(self, fname):
+        self.filename = fname
         self.basename = os.path.basename(fname)
+        self.name = self.basename[:-5]
         try:
             self.config = configparser.ConfigParser(interpolation=None)
             self.config.read(fname)

mesonbuild/wrap/wraptool.py

@@ -104,16 +104,24 @@ def install(options):
         f.write(data)
     print('Installed', name, 'branch', branch, 'revision', revision)
 
+def parse_patch_url(patch_url):
+    arr = patch_url.split('/')
+    return arr[-3], int(arr[-2])
+
 def get_current_version(wrapfile):
     cp = configparser.ConfigParser()
     cp.read(wrapfile)
     cp = cp['wrap-file']
     patch_url = cp['patch_url']
-    arr = patch_url.split('/')
-    branch = arr[-3]
-    revision = int(arr[-2])
+    branch, revision = parse_patch_url(patch_url)
     return branch, revision, cp['directory'], cp['source_filename'], cp['patch_filename']
 
+def update_wrap_file(wrapfile, name, new_branch, new_revision):
+    u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision))
+    data = u.read()
+    with open(wrapfile, 'wb') as f:
+        f.write(data)
+
 def update(options):
     name = options.name
     if not os.path.isdir('subprojects'):
@@ -128,8 +136,7 @@ def update(options):
     if new_branch == branch and new_revision == revision:
         print('Project', name, 'is already up to date.')
         sys.exit(0)
-    u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision))
-    data = u.read()
+    update_wrap_file(wrapfile, name, new_branch, new_revision)
     shutil.rmtree(os.path.join('subprojects', subdir), ignore_errors=True)
     try:
         os.unlink(os.path.join('subprojects/packagecache', src_file))
@@ -139,8 +146,6 @@ def update(options):
         os.unlink(os.path.join('subprojects/packagecache', patch_file))
     except FileNotFoundError:
         pass
-    with open(wrapfile, 'wb') as f:
-        f.write(data)
     print('Updated', name, 'to branch', new_branch, 'revision', new_revision)
 
 def info(options):