From 60b58e056fe6446310f21c68abfd9a7f5be37008 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 12 Oct 2018 13:53:05 -0400 Subject: [PATCH 1/5] Add 'meson subprojects update' command This is inspired by gst-build's git-update script. --- mesonbuild/mesonmain.py | 4 +- mesonbuild/mlog.py | 3 + mesonbuild/msubprojects.py | 179 ++++++++++++++++++++++++++++++++++++ mesonbuild/wrap/wrap.py | 2 + mesonbuild/wrap/wraptool.py | 19 ++-- 5 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 mesonbuild/msubprojects.py diff --git a/mesonbuild/mesonmain.py b/mesonbuild/mesonmain.py index ebe2c8e1c..c11d04410 100644 --- a/mesonbuild/mesonmain.py +++ b/mesonbuild/mesonmain.py @@ -20,7 +20,7 @@ import argparse from . import mesonlib from . import mlog -from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter +from . import mconf, minit, minstall, mintro, msetup, mtest, rewriter, msubprojects from .mesonlib import MesonException from .environment import detect_msys2_arch from .wrap import wraptool @@ -47,6 +47,8 @@ class CommandLineParser: help='Run tests') self.add_command('wrap', wraptool.add_arguments, wraptool.run, help='Wrap tools') + self.add_command('subprojects', msubprojects.add_arguments, msubprojects.run, + help='Manage subprojects') self.add_command('help', self.add_help_arguments, self.run_help_command, help='Print help of a subcommand') diff --git a/mesonbuild/mlog.py b/mesonbuild/mlog.py index 890cb46de..ea99d0971 100644 --- a/mesonbuild/mlog.py +++ b/mesonbuild/mlog.py @@ -96,6 +96,9 @@ def green(text): def yellow(text): return AnsiDecorator(text, "\033[1;33m") +def blue(text): + return AnsiDecorator(text, "\033[1;34m") + def cyan(text): return AnsiDecorator(text, "\033[1;36m") diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py new file mode 100644 index 000000000..6af8b41ef --- /dev/null +++ b/mesonbuild/msubprojects.py @@ -0,0 +1,179 @@ +import os, subprocess + +from . import mlog +from .mesonlib import Popen_safe +from .wrap.wrap import API_ROOT, PackageDefinition +from .wrap import wraptool + +def update_wrapdb_file(wrap, repo_dir, options): + patch_url = wrap.get('patch_url') + branch, revision = wraptool.parse_patch_url(patch_url) + new_branch, new_revision = wraptool.get_latest_version(wrap.name) + if new_branch == branch and new_revision == revision: + mlog.log(' -> Up to date.') + return + wraptool.update_wrap_file(wrap.filename, wrap.name, new_branch, new_revision) + msg = [' -> New wrap file downloaded.'] + # Meson reconfigure won't use the new wrap file as long as the source + # directory exists. We don't delete it ourself to avoid data loss in case + # user has changes in their copy. + if os.path.isdir(repo_dir): + msg += ['To use it, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure')] + mlog.log(*msg) + +def update_file(wrap, repo_dir, options): + patch_url = wrap.values.get('patch_url', '') + if patch_url.startswith(API_ROOT): + update_wrapdb_file(wrap, repo_dir, options) + elif not os.path.isdir(repo_dir): + # The subproject is not needed, or it is a tarball extracted in + # 'libfoo-1.0' directory and the version has been bumped and the new + # directory is 'libfoo-2.0'. In that case forcing a meson + # reconfigure will download and use the new tarball. + mlog.log(' -> Subproject has not been checked out. Run', mlog.bold('meson --reconfigure'), 'to fetch it if needed.') + else: + # The subproject has not changed, or the new source and/or patch + # tarballs should be extracted in the same directory than previous + # version. + mlog.log(' -> Subproject has not changed, or the new source/patch needs to be extracted on the same location.\n' + + ' In that case, delete', mlog.bold(repo_dir), 'and run', mlog.bold('meson --reconfigure')) + +def git(cmd, workingdir): + return subprocess.check_output(['git', '-C', workingdir] + cmd, + stderr=subprocess.STDOUT).decode() + +def update_git(wrap, repo_dir, options): + if not os.path.isdir(repo_dir): + mlog.log(' -> Not used.') + return + revision = wrap.get('revision') + ret = git(['rev-parse', '--abbrev-ref', 'HEAD'], repo_dir).strip() + if ret == 'HEAD': + try: + # We are currently in detached mode, just checkout the new revision + git(['fetch'], repo_dir) + git(['checkout', revision], repo_dir) + except subprocess.CalledProcessError as e: + out = e.output.decode().strip() + mlog.log(' -> Could not checkout revision', mlog.cyan(revision)) + mlog.log(mlog.red(out)) + mlog.log(mlog.red(str(e))) + return + elif ret == revision: + try: + # We are in the same branch, pull latest commits + git(['-c', 'rebase.autoStash=true', 'pull', '--rebase'], repo_dir) + except subprocess.CalledProcessError as e: + out = e.output.decode().strip() + mlog.log(' -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.') + mlog.log(mlog.red(out)) + mlog.log(mlog.red(str(e))) + return + else: + # We are in another branch, probably user created their own branch and + # we should rebase it on top of wrap's branch. + if options.rebase: + try: + git(['fetch'], repo_dir) + git(['-c', 'rebase.autoStash=true', 'rebase', revision], repo_dir) + except subprocess.CalledProcessError as e: + out = e.output.decode().strip() + mlog.log(' -> Could not rebase', mlog.bold(repo_dir), 'please fix and try again.') + mlog.log(mlog.red(out)) + mlog.log(mlog.red(str(e))) + return + else: + mlog.log(' -> Target revision is', mlog.bold(revision), 'but currently in branch is', mlog.bold(ret), '\n' + + ' To rebase your branch on top of', mlog.bold(revision), 'use', mlog.bold('--rebase'), 'option.') + return + + git(['submodule', 'update'], repo_dir) + commit_message = git(['show', '--quiet', '--pretty=format:%h%n%d%n%s%n[%an]'], repo_dir) + parts = [s.strip() for s in commit_message.split('\n')] + mlog.log(' ->', mlog.yellow(parts[0]), mlog.red(parts[1]), parts[2], mlog.blue(parts[3])) + +def update_hg(wrap, repo_dir, options): + if not os.path.isdir(repo_dir): + mlog.log(' -> Not used.') + return + revno = wrap.get('revision') + if revno.lower() == 'tip': + # Failure to do pull is not a fatal error, + # because otherwise you can't develop without + # a working net connection. + subprocess.call(['hg', 'pull'], cwd=repo_dir) + else: + if subprocess.call(['hg', 'checkout', revno], cwd=repo_dir) != 0: + subprocess.check_call(['hg', 'pull'], cwd=repo_dir) + subprocess.check_call(['hg', 'checkout', revno], cwd=repo_dir) + +def update_svn(wrap, repo_dir, options): + if not os.path.isdir(repo_dir): + mlog.log(' -> Not used.') + return + revno = wrap.get('revision') + p, out = Popen_safe(['svn', 'info', '--show-item', 'revision', repo_dir]) + current_revno = out + if current_revno == revno: + return + if revno.lower() == 'head': + # Failure to do pull is not a fatal error, + # because otherwise you can't develop without + # a working net connection. + subprocess.call(['svn', 'update'], cwd=repo_dir) + else: + subprocess.check_call(['svn', 'update', '-r', revno], cwd=repo_dir) + +def update(options): + src_dir = os.path.relpath(os.path.realpath(options.sourcedir)) + if not os.path.isfile(os.path.join(src_dir, 'meson.build')): + mlog.error('Directory', mlog.bold(src_dir), 'does not seem to be a Meson source directory.') + return 1 + subprojects_dir = os.path.join(src_dir, 'subprojects') + if not os.path.isdir(subprojects_dir): + mlog.log('Directory', mlog.bold(src_dir), 'does not seem to have subprojects.') + return 0 + files = [] + for name in options.subprojects: + f = os.path.join(subprojects_dir, name + '.wrap') + if not os.path.isfile(f): + mlog.error('Subproject', mlog.bold(name), 'not found.') + return 1 + else: + files.append(f) + if not files: + for f in os.listdir(subprojects_dir): + if f.endswith('.wrap'): + files.append(os.path.join(subprojects_dir, f)) + for f in files: + wrap = PackageDefinition(f) + directory = wrap.values.get('directory', wrap.name) + dirname = os.path.join(subprojects_dir, directory) + mlog.log('Updating %s...' % wrap.name) + if wrap.type == 'file': + update_file(wrap, dirname, options) + elif wrap.type == 'git': + update_git(wrap, dirname, options) + elif wrap.type == 'hg': + update_hg(wrap, dirname, options) + elif wrap.type == 'svn': + update_svn(wrap, dirname, options) + else: + mlog.log(' -> Cannot update', wrap.type, 'subproject') + return 0 + +def add_arguments(parser): + subparsers = parser.add_subparsers(title='Commands', dest='command') + subparsers.required = True + + p = subparsers.add_parser('update', help='Update all subprojects from wrap files') + p.add_argument('--sourcedir', default='.', + help='Path to source directory') + p.add_argument('--rebase', default=False, action='store_true', + help='Rebase your branch on top of wrap\'s revision (git only)') + p.add_argument('subprojects', nargs='*', + help='List of subprojects (default: all)') + p.set_defaults(subprojects_func=update) + +def run(options): + return options.subprojects_func(options) diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index 7cad90470..f4134d3b7 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -78,7 +78,9 @@ class WrapNotFoundException(WrapException): class PackageDefinition: def __init__(self, fname): + self.filename = fname self.basename = os.path.basename(fname) + self.name = self.basename[:-5] try: self.config = configparser.ConfigParser(interpolation=None) self.config.read(fname) diff --git a/mesonbuild/wrap/wraptool.py b/mesonbuild/wrap/wraptool.py index bb64b5b24..132decfde 100644 --- a/mesonbuild/wrap/wraptool.py +++ b/mesonbuild/wrap/wraptool.py @@ -104,16 +104,24 @@ def install(options): f.write(data) print('Installed', name, 'branch', branch, 'revision', revision) +def parse_patch_url(patch_url): + arr = patch_url.split('/') + return arr[-3], int(arr[-2]) + def get_current_version(wrapfile): cp = configparser.ConfigParser() cp.read(wrapfile) cp = cp['wrap-file'] patch_url = cp['patch_url'] - arr = patch_url.split('/') - branch = arr[-3] - revision = int(arr[-2]) + branch, revision = parse_patch_url(patch_url) return branch, revision, cp['directory'], cp['source_filename'], cp['patch_filename'] +def update_wrap_file(wrapfile, name, new_branch, new_revision): + u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision)) + data = u.read() + with open(wrapfile, 'wb') as f: + f.write(data) + def update(options): name = options.name if not os.path.isdir('subprojects'): @@ -128,8 +136,7 @@ def update(options): if new_branch == branch and new_revision == revision: print('Project', name, 'is already up to date.') sys.exit(0) - u = open_wrapdburl(API_ROOT + 'projects/%s/%s/%d/get_wrap' % (name, new_branch, new_revision)) - data = u.read() + update_wrap_file(wrapfile, name, new_branch, new_revision) shutil.rmtree(os.path.join('subprojects', subdir), ignore_errors=True) try: os.unlink(os.path.join('subprojects/packagecache', src_file)) @@ -139,8 +146,6 @@ def update(options): os.unlink(os.path.join('subprojects/packagecache', patch_file)) except FileNotFoundError: pass - with open(wrapfile, 'wb') as f: - f.write(data) print('Updated', name, 'to branch', new_branch, 'revision', new_revision) def info(options): From b5eac54c9e2641a486e698c7eeaf68981af55285 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Sun, 4 Nov 2018 22:13:06 -0500 Subject: [PATCH 2/5] Add 'meson subprojects checkout' command --- mesonbuild/msubprojects.py | 93 +++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py index 6af8b41ef..b9a4437f2 100644 --- a/mesonbuild/msubprojects.py +++ b/mesonbuild/msubprojects.py @@ -42,6 +42,11 @@ def git(cmd, workingdir): return subprocess.check_output(['git', '-C', workingdir] + cmd, stderr=subprocess.STDOUT).decode() +def git_show(repo_dir): + commit_message = git(['show', '--quiet', '--pretty=format:%h%n%d%n%s%n[%an]'], repo_dir) + parts = [s.strip() for s in commit_message.split('\n')] + mlog.log(' ->', mlog.yellow(parts[0]), mlog.red(parts[1]), parts[2], mlog.blue(parts[3])) + def update_git(wrap, repo_dir, options): if not os.path.isdir(repo_dir): mlog.log(' -> Not used.') @@ -88,9 +93,7 @@ def update_git(wrap, repo_dir, options): return git(['submodule', 'update'], repo_dir) - commit_message = git(['show', '--quiet', '--pretty=format:%h%n%d%n%s%n[%an]'], repo_dir) - parts = [s.strip() for s in commit_message.split('\n')] - mlog.log(' ->', mlog.yellow(parts[0]), mlog.red(parts[1]), parts[2], mlog.blue(parts[3])) + git_show(repo_dir) def update_hg(wrap, repo_dir, options): if not os.path.isdir(repo_dir): @@ -124,7 +127,59 @@ def update_svn(wrap, repo_dir, options): else: subprocess.check_call(['svn', 'update', '-r', revno], cwd=repo_dir) -def update(options): +def update(wrap, repo_dir, options): + mlog.log('Updating %s...' % wrap.name) + if wrap.type == 'file': + update_file(wrap, repo_dir, options) + elif wrap.type == 'git': + update_git(wrap, repo_dir, options) + elif wrap.type == 'hg': + update_hg(wrap, repo_dir, options) + elif wrap.type == 'svn': + update_svn(wrap, repo_dir, options) + else: + mlog.log(' -> Cannot update', wrap.type, 'subproject') + +def checkout(wrap, repo_dir, options): + if wrap.type != 'git' or not os.path.isdir(repo_dir): + return + branch_name = options.branch_name if options.branch_name else wrap.get('revision') + cmd = ['checkout', branch_name, '--'] + if options.b: + cmd.insert(1, '-b') + mlog.log('Checkout %s in %s...' % (branch_name, wrap.name)) + try: + git(cmd, repo_dir) + git_show(repo_dir) + except subprocess.CalledProcessError as e: + out = e.output.decode().strip() + mlog.log(' -> ', mlog.red(out)) + +def add_common_arguments(p): + p.add_argument('--sourcedir', default='.', + help='Path to source directory') + p.add_argument('subprojects', nargs='*', + help='List of subprojects (default: all)') + +def add_arguments(parser): + subparsers = parser.add_subparsers(title='Commands', dest='command') + subparsers.required = True + + p = subparsers.add_parser('update', help='Update all subprojects from wrap files') + p.add_argument('--rebase', default=False, action='store_true', + help='Rebase your branch on top of wrap\'s revision (git only)') + add_common_arguments(p) + p.set_defaults(subprojects_func=update) + + p = subparsers.add_parser('checkout', help='Checkout a branch (git only)') + p.add_argument('-b', default=False, action='store_true', + help='Create a new branch') + p.add_argument('branch_name', nargs='?', + help='Name of the branch to checkout or create (default: revision set in wrap file)') + add_common_arguments(p) + p.set_defaults(subprojects_func=checkout) + +def run(options): src_dir = os.path.relpath(os.path.realpath(options.sourcedir)) if not os.path.isfile(os.path.join(src_dir, 'meson.build')): mlog.error('Directory', mlog.bold(src_dir), 'does not seem to be a Meson source directory.') @@ -148,32 +203,6 @@ def update(options): for f in files: wrap = PackageDefinition(f) directory = wrap.values.get('directory', wrap.name) - dirname = os.path.join(subprojects_dir, directory) - mlog.log('Updating %s...' % wrap.name) - if wrap.type == 'file': - update_file(wrap, dirname, options) - elif wrap.type == 'git': - update_git(wrap, dirname, options) - elif wrap.type == 'hg': - update_hg(wrap, dirname, options) - elif wrap.type == 'svn': - update_svn(wrap, dirname, options) - else: - mlog.log(' -> Cannot update', wrap.type, 'subproject') + repo_dir = os.path.join(subprojects_dir, directory) + options.subprojects_func(wrap, repo_dir, options) return 0 - -def add_arguments(parser): - subparsers = parser.add_subparsers(title='Commands', dest='command') - subparsers.required = True - - p = subparsers.add_parser('update', help='Update all subprojects from wrap files') - p.add_argument('--sourcedir', default='.', - help='Path to source directory') - p.add_argument('--rebase', default=False, action='store_true', - help='Rebase your branch on top of wrap\'s revision (git only)') - p.add_argument('subprojects', nargs='*', - help='List of subprojects (default: all)') - p.set_defaults(subprojects_func=update) - -def run(options): - return options.subprojects_func(options) From d655515a6b8231600f37c6db48b802e7b58e070f Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 23 Nov 2018 14:55:06 -0500 Subject: [PATCH 3/5] Add 'meson subprojects download' command --- mesonbuild/msubprojects.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py index b9a4437f2..1536d963f 100644 --- a/mesonbuild/msubprojects.py +++ b/mesonbuild/msubprojects.py @@ -2,7 +2,7 @@ import os, subprocess from . import mlog from .mesonlib import Popen_safe -from .wrap.wrap import API_ROOT, PackageDefinition +from .wrap.wrap import API_ROOT, PackageDefinition, Resolver, WrapException from .wrap import wraptool def update_wrapdb_file(wrap, repo_dir, options): @@ -155,6 +155,18 @@ def checkout(wrap, repo_dir, options): out = e.output.decode().strip() mlog.log(' -> ', mlog.red(out)) +def download(wrap, repo_dir, options): + mlog.log('Download %s...' % wrap.name) + if os.path.isdir(repo_dir): + mlog.log(' -> Already downloaded') + return + try: + r = Resolver(os.path.dirname(repo_dir)) + r.resolve(wrap.name) + mlog.log(' -> done') + except WrapException as e: + mlog.log(' ->', mlog.red(str(e))) + def add_common_arguments(p): p.add_argument('--sourcedir', default='.', help='Path to source directory') @@ -179,6 +191,12 @@ def add_arguments(parser): add_common_arguments(p) p.set_defaults(subprojects_func=checkout) + p = subparsers.add_parser('download', help='Ensure subprojects are fetched, even if not in use. ' + + 'Already downloaded subprojects are not modified. ' + + 'This can be used to pre-fetch all subprojects and avoid downloads during configure.') + add_common_arguments(p) + p.set_defaults(subprojects_func=download) + def run(options): src_dir = os.path.relpath(os.path.realpath(options.sourcedir)) if not os.path.isfile(os.path.join(src_dir, 'meson.build')): From 0e7513e734617ab60ec0edc96e046029e983ea80 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 23 Nov 2018 15:40:33 -0500 Subject: [PATCH 4/5] flake8: ignore W504: line break after binary operator It wasn't an error before, and plenty of files does that. --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 65a3c64c8..7a94d8535 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,4 +26,6 @@ ignore = E741 # E722: do not use bare except' E722 + # W504: line break after binary operator + W504 max-line-length = 120 From 2efedf80e0129ee1dea560aa652f6358e3686854 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Mon, 26 Nov 2018 16:42:55 -0500 Subject: [PATCH 5/5] Add documentation for 'meson subprojects' command line --- docs/markdown/Subprojects.md | 47 ++++++++++++++++++++++- docs/markdown/snippets/subprojects_cmd.md | 7 ++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 docs/markdown/snippets/subprojects_cmd.md diff --git a/docs/markdown/Subprojects.md b/docs/markdown/Subprojects.md index 2e3e2ead5..ed118522e 100644 --- a/docs/markdown/Subprojects.md +++ b/docs/markdown/Subprojects.md @@ -179,7 +179,7 @@ the following command-line options: This is useful (mostly for distros) when you want to only use the sources provided by a software release, and want to manually handle or provide missing dependencies. - + * **--wrap-mode=nofallback** Meson will not use subproject fallbacks for any dependency @@ -196,6 +196,51 @@ the following command-line options: want to specifically build against the library sources provided by your subprojects. +## Download subprojects + +*Since 0.49.0* + +Meson will automatically download needed subprojects during configure, unless +**--wrap-mode=nodownload** option is passed. It is sometimes preferable to +download all subprojects in advance, so the meson configure can be performed +offline. The command-line `meson subprojects download` can be used for that, it +will download all missing subprojects, but will not update already fetched +subprojects. + +## Update subprojects + +*Since 0.49.0* + +Once a subproject has been fetched, Meson will not update it automatically. +For example if the wrap file tracks a git branch, it won't pull latest commits. + +To pull latest version of all your subprojects at once, just run the command: +`meson subprojects update`. +- If the wrap file comes from wrapdb, the latest version of the wrap file will + be pulled and used next time meson reconfigure the project. This can be + triggered using `meson --reconfigure`. Previous source tree is not deleted, to + prevent from any loss of local changes. +- If the wrap file points to a git commit or tag, a checkout of that commit is + performed. +- If the wrap file points to a git branch, and the current branch has the same + name, a `git pull` is performed. +- If the wrap file points to a git branch, and the current branch is different, + it is skipped. Unless `--rebase` option is passed in which case + `git pull --rebase` is performed. + +## Start a topic branch across all git subprojects + +*Since 0.49.0* + +The command-line `meson subprojects checkout ` will checkout a +branch, or create one with `-b` argument, in every git subprojects. This is +useful when starting local changes across multiple subprojects. It is still your +responsability to commit and push in each repository where you made local +changes. + +To come back to the revision set in wrap file (i.e. master), just run +`meson subprojects checkout` with no branch name. + ## Why must all subprojects be inside a single directory? There are several reasons. diff --git a/docs/markdown/snippets/subprojects_cmd.md b/docs/markdown/snippets/subprojects_cmd.md new file mode 100644 index 000000000..20fef5cad --- /dev/null +++ b/docs/markdown/snippets/subprojects_cmd.md @@ -0,0 +1,7 @@ +## Subprojects download, checkout, update command-line + +New command-line tool has been added to manage subprojects: + +- `meson subprojects download` to download all subprojects that have a wrap file. +- `meson subprojects update` to update all subprojects to latest version. +- `meson subprojects checkout` to checkout or create a branch in all git subprojects.