Merge pull request #1669 from mesonbuild/dist

Create a dist target

docs/markdown/Creating-releases.md

@@ -0,0 +1,15 @@
+---
+short-description: Creating releases
+...
+
+# Creating releases
+
+In addition to development, almost all projects provide periodical source releases. These are standalone packages (usually either in tar or zip format) of the source code. They do not contain any revision control metadata, only the source code.
+
+Meson provides a simple way of generating these. It consists of a single command:
+
+    ninja dist
+
+This creates a file called `projectname-version.tar.xz` in the build tree subdirectory `meson-dist`. This archive contains the full contents of the latest commit in revision control including all the submodules. All revision control metadata is removed. Meson then takes this archive and tests that it works by doing a full compile + test + install cycle. If all these pass, Meson will then create a SHA-256 checksum file next to the archive.
+
+**Note**: Meson behaviour is different from Autotools. The Autotools "dist" target packages up the current source tree. Meson packages the latest revision control commit. The reason for this is that it prevents developers from doing accidental releases where the distributed archive does not match any commit in revision control (especially the one tagged for the release).

docs/markdown/Release-notes-for-0.41.0.md

@@ -38,3 +38,18 @@ pkg.generate(libraries : libs,
              description : 'A simple demo library.',
              variables : ['datadir=${prefix}/data'])
 ```
+
+## A target for creating tarballs
+
+Creating distribution tarballs is simple:
+
+    ninja dist
+
+This will create a `.tar.xz` archive of the source code including
+submodules without any revision control information. This command also
+verifies that the resulting archive can be built, tested and
+installed. This is roughly equivalent to the `distcheck` target in
+other build systems. Currently this only works for projects using Git
+and only with the Ninja backend.
+
+

docs/sitemap.txt

@@ -42,6 +42,7 @@ index.md
 		Build-system-converters.md
 		Configuring-a-build-directory.md
 		Run-targets.md
+		Creating-releases.md
 		Creating-OSX-packages.md
 		Creating-Linux-binaries.md
 	Reference-manual.md

mesonbuild/backend/ninjabackend.py

@@ -195,6 +195,7 @@ int dummy;
             self.generate_tests(outfile)
             outfile.write('# Install rules\n\n')
             self.generate_install(outfile)
+            self.generate_dist(outfile)
             if 'b_coverage' in self.environment.coredata.base_options and \
                     self.environment.coredata.base_options['b_coverage'].value:
                 outfile.write('# Coverage rules\n\n')
@@ -2375,6 +2376,19 @@ rule FORTRAN_DEP_HACK
         # affect behavior in any other way.
         return sorted(cmds)
 
+    def generate_dist(self, outfile):
+        elem = NinjaBuildElement(self.all_outputs, 'dist', 'CUSTOM_COMMAND', 'PHONY')
+        elem.add_item('DESC', 'Creating source packages')
+        elem.add_item('COMMAND', [sys.executable,
+                                  self.environment.get_build_command(),
+                                  '--internal', 'dist',
+                                  self.environment.source_dir,
+                                  self.environment.build_dir,
+                                  sys.executable,
+                                  self.environment.get_build_command()])
+        elem.add_item('pool', 'console')
+        elem.write(outfile)
+
     # For things like scan-build and other helper tools we might have.
     def generate_utils(self, outfile):
         cmd = [sys.executable, self.environment.get_build_command(),

mesonbuild/coredata.py

@@ -347,4 +347,6 @@ forbidden_target_names = {'clean': None,
                           'build.ninja': None,
                           'scan-build': None,
                           'reconfigure': None,
+                          'dist': None,
+                          'distcheck': None,
                           }

mesonbuild/mesonmain.py

@@ -246,6 +246,9 @@ def run_script_command(args):
     elif cmdname == 'uninstall':
         import mesonbuild.scripts.uninstall as abc
         cmdfunc = abc.run
+    elif cmdname == 'dist':
+        import mesonbuild.scripts.dist as abc
+        cmdfunc = abc.run
     else:
         raise MesonException('Unknown internal command {}.'.format(cmdname))
     return cmdfunc(cmdargs)

mesonbuild/scripts/dist.py

@@ -0,0 +1,148 @@
+# Copyright 2017 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os, sys
+import shutil
+import argparse
+import subprocess
+import pickle
+import hashlib
+import tarfile, zipfile
+import tempfile
+from glob import glob
+from mesonbuild.environment import detect_ninja
+
+def create_hash(fname):
+    hashname = fname + '.sha256sum'
+    m = hashlib.sha256()
+    m.update(open(fname, 'rb').read())
+    with open(hashname, 'w') as f:
+        f.write('%s %s\n' % (m.hexdigest(), os.path.split(fname)[-1]))
+
+def create_zip(zipfilename, packaging_dir):
+    prefix = os.path.split(packaging_dir)[0]
+    removelen = len(prefix) + 1
+    with zipfile.ZipFile(zipfilename,
+                         'w',
+                         compression=zipfile.ZIP_DEFLATED,
+                         allowZip64=True) as zf:
+        zf.write(packaging_dir, packaging_dir[removelen:])
+        for root, dirs, files in os.walk(packaging_dir):
+            for d in dirs:
+                dname = os.path.join(root, d)
+                zf.write(dname, dname[removelen:])
+            for f in files:
+                fname = os.path.join(root, f)
+                zf.write(fname, fname[removelen:])
+
+def del_gitfiles(dirname):
+    for f in glob(os.path.join(dirname, '.git*')):
+        if os.path.isdir(f) and not os.path.islink(f):
+            shutil.rmtree(f)
+        else:
+            os.unlink(f)
+
+def process_submodules(dirname):
+    module_file = os.path.join(dirname, '.gitmodules')
+    if not os.path.exists(module_file):
+        return
+    subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=dirname)
+    for line in open(module_file):
+        line = line.strip()
+        if '=' not in line:
+            continue
+        k, v = line.split('=', 1)
+        k = k.strip()
+        v = v.strip()
+        if k != 'path':
+            continue
+        del_gitfiles(os.path.join(dirname, v))
+
+def create_dist(dist_name, src_root, bld_root, dist_sub):
+    distdir = os.path.join(dist_sub, dist_name)
+    if os.path.exists(distdir):
+        shutil.rmtree(distdir)
+    os.makedirs(distdir)
+    subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
+    process_submodules(distdir)
+    del_gitfiles(distdir)
+    xzname = distdir + '.tar.xz'
+    # Should use shutil but it got xz support only in 3.5.
+    with tarfile.open(xzname, 'w:xz') as tf:
+        tf.add(distdir, os.path.split(distdir)[1])
+    # Create only .tar.xz for now.
+    #zipname = distdir + '.zip'
+    #create_zip(zipname, distdir)
+    shutil.rmtree(distdir)
+    return (xzname, )
+
+def check_dist(packagename, meson_command):
+    print('Testing distribution package %s.' % packagename)
+    unpackdir = tempfile.mkdtemp()
+    builddir = tempfile.mkdtemp()
+    installdir = tempfile.mkdtemp()
+    ninja_bin = detect_ninja()
+    try:
+        tf = tarfile.open(packagename)
+        tf.extractall(unpackdir)
+        srcdir = glob(os.path.join(unpackdir, '*'))[0]
+        if subprocess.call(meson_command + ['--backend=ninja', srcdir, builddir]) != 0:
+            print('Running Meson on distribution package failed')
+            return 1
+        if subprocess.call([ninja_bin], cwd=builddir) != 0:
+            print('Compiling the distribution package failed.')
+            return 1
+        if subprocess.call([ninja_bin, 'test'], cwd=builddir) != 0:
+            print('Running unit tests on the distribution package failed.')
+            return 1
+        myenv = os.environ.copy()
+        myenv['DESTDIR'] = installdir
+        if subprocess.call([ninja_bin, 'install'], cwd=builddir, env=myenv) != 0:
+            print('Installing the distribution package failed.')
+            return 1
+    finally:
+        shutil.rmtree(srcdir)
+        shutil.rmtree(builddir)
+        shutil.rmtree(installdir)
+    print('Distribution package %s tested.' % packagename)
+    return 0
+
+def run(args):
+    src_root = args[0]
+    bld_root = args[1]
+    meson_command = args[2:]
+    priv_dir = os.path.join(bld_root, 'meson-private')
+    dist_sub = os.path.join(bld_root, 'meson-dist')
+
+    buildfile = os.path.join(priv_dir, 'build.dat')
+
+    build = pickle.load(open(buildfile, 'rb'))
+
+    dist_name = build.project_name + '-' + build.project_version
+
+    if not os.path.isdir(os.path.join(src_root, '.git')):
+        print('Dist currently only works with Git repos.')
+        return 1
+    names = create_dist(dist_name, src_root, bld_root, dist_sub)
+    if names is None:
+        return 1
+    error_count = 0
+    for name in names:
+        rc = check_dist(name, meson_command) # Check only one.
+        rc = 0
+        if rc == 0:
+            create_hash(name)
+        error_count += rc
+    return rc

run_unittests.py

@@ -331,6 +331,7 @@ class BasePlatformTests(unittest.TestCase):
         self.prefix = '/usr'
         self.libdir = os.path.join(self.prefix, 'lib')
         self.installdir = os.path.join(self.builddir, 'install')
+        self.distdir = os.path.join(self.builddir, 'meson-dist')
         # Get the backend
         # FIXME: Extract this from argv?
         self.backend = getattr(Backend, os.environ.get('MESON_UNIT_TEST_BACKEND', 'ninja'))
@@ -1065,6 +1066,47 @@ class AllPlatformTests(BasePlatformTests):
         self.build()
         self.run_tests()
 
+    def test_dist(self):
+        if not shutil.which('git'):
+            raise unittest.SkipTest('Git not found')
+        try:
+            self.dist_impl()
+        except PermissionError:
+            # When run under Windows CI, something (virus scanner?)
+            # holds on to the git files so cleaning up the dir
+            # fails sometimes.
+            pass
+
+    def dist_impl(self):
+        # Create this on the fly because having rogue .git directories inside
+        # the source tree leads to all kinds of trouble.
+        with tempfile.TemporaryDirectory() as project_dir:
+            with open(os.path.join(project_dir, 'meson.build'), 'w') as ofile:
+                ofile.write('''project('disttest', 'c', version : '1.4.3')
+e = executable('distexe', 'distexe.c')
+test('dist test', e)
+''')
+            with open(os.path.join(project_dir, 'distexe.c'), 'w') as ofile:
+                ofile.write('''#include<stdio.h>
+
+int main(int argc, char **argv) {
+    printf("I am a distribution test.\\n");
+    return 0;
+}
+''')
+            subprocess.check_call(['git', 'init'], cwd=project_dir)
+            subprocess.check_call(['git', 'config',
+                                   'user.name', 'Author Person'], cwd=project_dir)
+            subprocess.check_call(['git', 'config',
+                                   'user.email', 'teh_coderz@example.com'], cwd=project_dir)
+            subprocess.check_call(['git', 'add', 'meson.build', 'distexe.c'], cwd=project_dir)
+            subprocess.check_call(['git', 'commit', '-a', '-m', 'I am a project'], cwd=project_dir)
+            self.init(project_dir)
+            self.build('dist')
+            distfile = os.path.join(self.distdir, 'disttest-1.4.3.tar.xz')
+            checksumfile = distfile + '.sha256sum'
+            self.assertTrue(os.path.exists(distfile))
+            self.assertTrue(os.path.exists(checksumfile))
 
 class WindowsTests(BasePlatformTests):
     '''