From d6fba7f01ce3effebd18d75ebe3102774353cee0 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Fri, 5 Oct 2018 12:07:46 -0400 Subject: [PATCH] wrap: Allow source and patch to be local files It is sometimes important to be able to build projects offline, in that case subproject tarballs and patches could be shipped directly within the project's repository. --- .../markdown/Wrap-dependency-system-manual.md | 10 ++ mesonbuild/wrap/wrap.py | 165 +++++++++--------- .../meson.build | 4 +- 3 files changed, 91 insertions(+), 88 deletions(-) diff --git a/docs/markdown/Wrap-dependency-system-manual.md b/docs/markdown/Wrap-dependency-system-manual.md index 2e977b2ea..7da4be541 100644 --- a/docs/markdown/Wrap-dependency-system-manual.md +++ b/docs/markdown/Wrap-dependency-system-manual.md @@ -45,6 +45,11 @@ If you then use this subproject in your build, Meson will automatically download and extract it during build. This makes subproject embedding extremely easy. +Since *0.49.0* if `source_filename` is found in project's +`subprojects/packagecache` directory, it will be used instead of downloading the +source, even if `wrap-mode` option is set to `nodownload`. The file's hash will +be checked. + Unfortunately most software projects in the world do not build with Meson. Because of this Meson allows you to specify a patch URL. This works in much the same way as Debian's distro patches. That is, they @@ -76,6 +81,11 @@ thousands of lines of code. Once you have a working build definition, just zip up the Meson build files (and others you have changed) and put them somewhere where you can download them. +Since *0.49.0* if `patch_filename` is found in project's +`subprojects/packagecache` directory, it will be used instead of downloading the +patch, even if `wrap-mode` option is set to `nodownload`. The file's hash will +be checked. + ## Branching subprojects directly from git The above mentioned scheme assumes that your subproject is working off diff --git a/mesonbuild/wrap/wrap.py b/mesonbuild/wrap/wrap.py index be8b04fd6..a0eeed1a9 100644 --- a/mesonbuild/wrap/wrap.py +++ b/mesonbuild/wrap/wrap.py @@ -92,9 +92,10 @@ class Resolver: self.cachedir = os.path.join(self.subdir_root, 'packagecache') def resolve(self, packagename): + self.packagename = packagename # We always have to load the wrap file, if it exists, because it could # override the default directory name. - p = self.load_wrap(packagename) + p = self.load_wrap() directory = packagename if p and 'directory' in p.values: directory = p.get('directory') @@ -114,30 +115,23 @@ class Resolver: m = '{!r} already exists and is not a dir; cannot use as subproject' raise RuntimeError(m.format(subprojdir)) else: - # Don't download subproject data based on wrap file if requested. - # Git submodules are ok (see above)! - if self.wrap_mode is WrapMode.nodownload: - m = 'Automatic wrap-based subproject downloading is disabled' - raise RuntimeError(m) - # A wrap file is required to download if not p: m = 'No {}.wrap found for {!r}' raise RuntimeError(m.format(packagename, subprojdir)) if p.type == 'file': - if not os.path.isdir(self.cachedir): - os.mkdir(self.cachedir) - self.download(p, packagename) - self.extract_package(p) - elif p.type == 'git': - self.get_git(p) - elif p.type == "hg": - self.get_hg(p) - elif p.type == "svn": - self.get_svn(p) + self.get_file(p) else: - raise AssertionError('Unreachable code.') + self.check_can_download() + if p.type == 'git': + self.get_git(p) + elif p.type == "hg": + self.get_hg(p) + elif p.type == "svn": + self.get_svn(p) + else: + raise AssertionError('Unreachable code.') # A meson.build file is required in the directory if not os.path.exists(meson_file): @@ -146,12 +140,19 @@ class Resolver: return directory - def load_wrap(self, packagename): - fname = os.path.join(self.subdir_root, packagename + '.wrap') + def load_wrap(self): + fname = os.path.join(self.subdir_root, self.packagename + '.wrap') if os.path.isfile(fname): return PackageDefinition(fname) return None + def check_can_download(self): + # Don't download subproject data based on wrap file if requested. + # Git submodules are ok (see above)! + if self.wrap_mode is WrapMode.nodownload: + m = 'Automatic wrap-based subproject downloading is disabled' + raise RuntimeError(m) + def resolve_git_submodule(self, dirname): # Are we in a git repository? ret, out = quiet_git(['rev-parse'], self.subdir_root) @@ -184,6 +185,22 @@ class Resolver: m = 'Unknown git submodule output: {!r}' raise RuntimeError(m.format(out)) + def get_file(self, p): + path = self.get_file_internal(p, 'source') + target_dir = os.path.join(self.subdir_root, p.get('directory')) + extract_dir = self.subdir_root + # Some upstreams ship packages that do not have a leading directory. + # Create one for them. + try: + p.get('lead_directory_missing') + os.mkdir(target_dir) + extract_dir = target_dir + except KeyError: + pass + shutil.unpack_archive(path, extract_dir) + if p.has_patch(): + self.apply_patch(p) + def get_git(self, p): checkoutdir = os.path.join(self.subdir_root, p.get('directory')) revno = p.get('revision') @@ -312,41 +329,48 @@ class Resolver: hashvalue = h.hexdigest() return hashvalue, tmpfile.name - def get_hash(self, data): + def check_hash(self, p, what, path): + expected = p.get(what + '_hash') h = hashlib.sha256() - h.update(data) - hashvalue = h.hexdigest() - return hashvalue - - def download(self, p, packagename): - ofname = os.path.join(self.cachedir, p.get('source_filename')) - if os.path.exists(ofname): - mlog.log('Using', mlog.bold(packagename), 'from cache.') - else: - srcurl = p.get('source_url') - mlog.log('Downloading', mlog.bold(packagename), 'from', mlog.bold(srcurl)) - dhash, tmpfile = self.get_data(srcurl) - expected = p.get('source_hash') - if dhash != expected: - os.remove(tmpfile) - raise RuntimeError('Incorrect hash for source %s:\n %s expected\n %s actual.' % (packagename, expected, dhash)) - os.rename(tmpfile, ofname) - if p.has_patch(): - patch_filename = p.get('patch_filename') - filename = os.path.join(self.cachedir, patch_filename) - if os.path.exists(filename): - mlog.log('Using', mlog.bold(patch_filename), 'from cache.') - else: - purl = p.get('patch_url') - mlog.log('Downloading patch from', mlog.bold(purl)) - phash, tmpfile = self.get_data(purl) - expected = p.get('patch_hash') - if phash != expected: - os.remove(tmpfile) - raise RuntimeError('Incorrect hash for patch %s:\n %s expected\n %s actual' % (packagename, expected, phash)) - os.rename(tmpfile, filename) - else: - mlog.log('Package does not require patch.') + with open(path, 'rb') as f: + h.update(f.read()) + dhash = h.hexdigest() + if dhash != expected: + raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) + + def download(self, p, what, ofname): + self.check_can_download() + srcurl = p.get(what + '_url') + mlog.log('Downloading', mlog.bold(self.packagename), what, 'from', mlog.bold(srcurl)) + dhash, tmpfile = self.get_data(srcurl) + expected = p.get(what + '_hash') + if dhash != expected: + os.remove(tmpfile) + raise RuntimeError('Incorrect hash for %s:\n %s expected\n %s actual.' % (what, expected, dhash)) + os.rename(tmpfile, ofname) + + def get_file_internal(self, p, what): + filename = p.get(what + '_filename') + cache_path = os.path.join(self.cachedir, filename) + + if os.path.exists(cache_path): + self.check_hash(p, what, cache_path) + mlog.log('Using', mlog.bold(self.packagename), what, 'from cache.') + return cache_path + + if not os.path.isdir(self.cachedir): + os.mkdir(self.cachedir) + self.download(p, what, cache_path) + return cache_path + + def apply_patch(self, p): + path = self.get_file_internal(p, 'patch') + try: + shutil.unpack_archive(path, self.subdir_root) + except Exception: + with tempfile.TemporaryDirectory() as workdir: + shutil.unpack_archive(path, workdir) + self.copy_tree(workdir, self.subdir_root) def copy_tree(self, root_src_dir, root_dst_dir): """ @@ -366,36 +390,3 @@ class Resolver: os.chmod(dst_file, stat.S_IWUSR) os.remove(dst_file) shutil.copy2(src_file, dst_dir) - - def extract_package(self, package): - if sys.version_info < (3, 5): - try: - import lzma # noqa: F401 - del lzma - except ImportError: - pass - else: - try: - shutil.register_unpack_format('xztar', ['.tar.xz', '.txz'], shutil._unpack_tarfile, [], "xz'ed tar-file") - except shutil.RegistryError: - pass - target_dir = os.path.join(self.subdir_root, package.get('directory')) - if os.path.isdir(target_dir): - return - extract_dir = self.subdir_root - # Some upstreams ship packages that do not have a leading directory. - # Create one for them. - try: - package.get('lead_directory_missing') - os.mkdir(target_dir) - extract_dir = target_dir - except KeyError: - pass - shutil.unpack_archive(os.path.join(self.cachedir, package.get('source_filename')), extract_dir) - if package.has_patch(): - try: - shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), self.subdir_root) - except Exception: - with tempfile.TemporaryDirectory() as workdir: - shutil.unpack_archive(os.path.join(self.cachedir, package.get('patch_filename')), workdir) - self.copy_tree(workdir, self.subdir_root) diff --git a/test cases/common/158 wrap file should not failed/meson.build b/test cases/common/158 wrap file should not failed/meson.build index 9cf4e9a6b..9d707eec5 100644 --- a/test cases/common/158 wrap file should not failed/meson.build +++ b/test cases/common/158 wrap file should not failed/meson.build @@ -1,4 +1,6 @@ -project('mainproj', 'c') +project('mainproj', 'c', + default_options : ['wrap_mode=nodownload'], +) subproject('zlib')