From b6d277c140c7cbec3349bf5bd5986fc79f804e42 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 12 Mar 2021 00:24:21 -0600 Subject: [PATCH] Add 'subprojects purge' command This will help facilitate cache busting in certain situations, and replaces hand-rolled solutions of writing a length command to remove various files/folders within the subprojects directory. --- docs/markdown/snippets/subprojects_purge.md | 18 ++++++ mesonbuild/msubprojects.py | 64 ++++++++++++++++++++- run_unittests.py | 31 +++++++++- 3 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 docs/markdown/snippets/subprojects_purge.md diff --git a/docs/markdown/snippets/subprojects_purge.md b/docs/markdown/snippets/subprojects_purge.md new file mode 100644 index 000000000..86e1064b2 --- /dev/null +++ b/docs/markdown/snippets/subprojects_purge.md @@ -0,0 +1,18 @@ +## Purge subprojects folder + +It is now possible to purge a subprojects folder of artifacts created +from wrap-based subprojects including anything in `packagecache`. This is useful +when you want to return to a completely clean source tree or busting caches with +stale patch directories or caches. By default the command will only print out +what it is removing. You need to pass `--confirm` to the command for actual +artifacts to be purged. + +By default all wrap-based subprojects will be purged. + +- `meson subprojects purge` prints non-cache wrap artifacts which will be +purged. +- `meson subprojects purge --confirm` purges non-cache wrap artifacts. +- `meson subprojects purge --confirm --include-cache` also removes the cache +artifacts. +- `meson subprojects purge --confirm subproj1 subproj2` removes non-cache wrap +artifacts associated with the listed subprojects. diff --git a/mesonbuild/msubprojects.py b/mesonbuild/msubprojects.py index d038fa734..bdd8c1fc2 100755 --- a/mesonbuild/msubprojects.py +++ b/mesonbuild/msubprojects.py @@ -4,7 +4,7 @@ from pathlib import Path from . import mlog from .mesonlib import quiet_git, verbose_git, GitException, Popen_safe, MesonException, windows_proof_rmtree -from .wrap.wrap import API_ROOT, Resolver, WrapException, ALL_TYPES +from .wrap.wrap import API_ROOT, PackageDefinition, Resolver, WrapException, ALL_TYPES from .wrap import wraptool ALL_TYPES_STRING = ', '.join(ALL_TYPES) @@ -312,6 +312,61 @@ def foreach(r, wrap, repo_dir, options): mlog.log(out, end='') return True +def purge(r: Resolver, wrap: PackageDefinition, repo_dir: str, options: argparse.Namespace) -> bool: + # if subproject is not wrap-based, then don't remove it + if not wrap.type: + return True + + if wrap.type == 'redirect': + redirect_file = Path(wrap.filename).resolve() + if options.confirm: + redirect_file.unlink() + mlog.log(f'Deleting {redirect_file}') + + if options.include_cache: + packagecache = Path(r.cachedir).resolve() + subproject_cache_file = packagecache / wrap.get("source_filename") + if subproject_cache_file.is_file(): + if options.confirm: + subproject_cache_file.unlink() + mlog.log(f'Deleting {subproject_cache_file}') + + try: + subproject_patch_file = packagecache / wrap.get("patch_filename") + if subproject_patch_file.is_file(): + if options.confirm: + subproject_patch_file.unlink() + mlog.log(f'Deleting {subproject_patch_file}') + except WrapException: + pass + + # Don't log that we will remove an empty directory + if packagecache.exists() and not any(packagecache.iterdir()): + packagecache.rmdir() + + subproject_source_dir = Path(repo_dir).resolve() + + # Don't follow symlink. This is covered by the next if statement, but why + # not be doubly sure. + if subproject_source_dir.is_symlink(): + if options.confirm: + subproject_source_dir.unlink() + mlog.log(f'Deleting {subproject_source_dir}') + return True + + if not subproject_source_dir.is_dir(): + return True + + try: + if options.confirm: + windows_proof_rmtree(str(subproject_source_dir)) + mlog.log(f'Deleting {subproject_source_dir}') + except OSError as e: + mlog.error(f'Unable to remove: {subproject_source_dir}: {e}') + return False + + return True + def add_common_arguments(p): p.add_argument('--sourcedir', default='.', help='Path to source directory') @@ -361,6 +416,13 @@ def add_arguments(parser): p.set_defaults(subprojects=[]) p.set_defaults(subprojects_func=foreach) + p = subparsers.add_parser('purge', help='Remove all wrap-based subproject artifacts') + add_common_arguments(p) + add_subprojects_argument(p) + p.add_argument('--include-cache', action='store_true', default=False, help='Remove the package cache as well') + p.add_argument('--confirm', action='store_true', default=False, help='Confirm the removal of subproject artifacts') + p.set_defaults(subprojects_func=purge) + def run(options): src_dir = os.path.relpath(os.path.realpath(options.sourcedir)) if not os.path.isfile(os.path.join(src_dir, 'meson.build')): diff --git a/run_unittests.py b/run_unittests.py index b6e349d32..6756c80ee 100755 --- a/run_unittests.py +++ b/run_unittests.py @@ -9675,6 +9675,8 @@ class SubprojectsCommandTests(BasePlatformTests): self.subprojects_dir = self.project_dir / 'subprojects' os.makedirs(str(self.subprojects_dir)) + self.packagecache_dir = self.subprojects_dir / 'packagecache' + os.makedirs(str(self.packagecache_dir)) def _create_project(self, path, project_name='dummy'): os.makedirs(str(path), exist_ok=True) @@ -9752,10 +9754,12 @@ class SubprojectsCommandTests(BasePlatformTests): path = self.root_dir / tarball with open(str((self.subprojects_dir / name).with_suffix('.wrap')), 'w') as f: f.write(textwrap.dedent( - ''' + f''' [wrap-file] - source_url={} - '''.format(os.path.abspath(str(path))))) + source_url={os.path.abspath(str(path))} + source_filename={tarball} + ''')) + Path(self.packagecache_dir / tarball).touch() def _subprojects_cmd(self, args): return self._run(self.meson_command + ['subprojects'] + args, workdir=str(self.project_dir)) @@ -9864,6 +9868,27 @@ class SubprojectsCommandTests(BasePlatformTests): out = self._subprojects_cmd(['foreach', '--types', 'git'] + dummy_cmd) self.assertEqual(ran_in(out), ['subprojects/sub_git']) + def test_purge(self): + self._create_project(self.subprojects_dir / 'sub_file') + self._wrap_create_file('sub_file') + + def deleting(s) -> T.List[str]: + ret = [] + prefix = 'Deleting ' + for l in s.splitlines(): + if l.startswith(prefix): + ret.append(l[len(prefix):]) + return sorted(ret) + + out = self._subprojects_cmd(['purge']) + self.assertEqual(deleting(out), [str(self.subprojects_dir / 'sub_file')]) + out = self._subprojects_cmd(['purge', '--include-cache']) + self.assertEqual(deleting(out), [str(self.subprojects_dir / 'packagecache' / 'dummy.tar.gz'), str(self.subprojects_dir / 'sub_file')]) + out = self._subprojects_cmd(['purge', '--include-cache', '--confirm']) + self.assertEqual(deleting(out), [str(self.subprojects_dir / 'packagecache' / 'dummy.tar.gz'), str(self.subprojects_dir / 'sub_file')]) + self.assertFalse(Path(self.subprojects_dir / 'packagecache' / 'dummy.tar.gz').exists()) + self.assertFalse(Path(self.subprojects_dir / 'sub_file').exists()) + def _clang_at_least(compiler: 'Compiler', minver: str, apple_minver: T.Optional[str]) -> bool: """ check that Clang compiler is at least a specified version, whether AppleClang or regular Clang