dist: Allow packaging subproject in same git repo as main project

docs/markdown/Creating-releases.md

@@ -58,3 +58,21 @@ for example when done in CI that already does its own testing.
 So with `--no-tests` you can tell Meson "Do not build and test generated
 packages.".
 
+## Release a subproject separately
+
+*Since 0.57.0* the `meson dist` command can now create a distribution tarball
+for a subproject in the same git repository as the main project. This can be
+useful if parts of the project (e.g. libraries) can be built and distributed
+separately. In that case they can be moved into `subprojects/mysub` and running
+`meson dist` in that directory will now create a tarball containing only the
+source code from that subdir and not the rest of the main project or other
+subprojects.
+
+For example:
+```sh
+git clone https://github.com/myproject
+cd myproject/subprojects/mysubproject
+meson builddir
+meson dist -C builddir
+```
+This produces `builddir/meson-dist/mysubproject-1.0.tar.xz` tarball.

docs/markdown/snippets/dist_subproject.md

@@ -0,0 +1,16 @@
+## Package a subproject
+
+The `meson dist` command can now create a distribution tarball for a subproject
+in the same git repository as the main project. This can be useful if parts of
+the project (e.g. libraries) can be built and distributed separately. In that
+case they can be moved into `subprojects/mysub` and running `meson dist` in that
+directory will now create a tarball containing only the source code from that
+subdir and not the rest of the main project or other subprojects.
+
+For example:
+```sh
+git clone https://github.com/myproject
+cd myproject/subprojects/mysubproject
+meson builddir
+meson dist -C builddir
+```

mesonbuild/mdist.py

@@ -23,7 +23,7 @@ import json
 from glob import glob
 from pathlib import Path
 from mesonbuild.environment import detect_ninja
-from mesonbuild.mesonlib import windows_proof_rmtree, MesonException
+from mesonbuild.mesonlib import windows_proof_rmtree, MesonException, quiet_git
 from mesonbuild.wrap import wrap
 from mesonbuild import mlog, build
 
@@ -96,9 +96,21 @@ def run_dist_scripts(src_root, bld_root, dist_root, dist_scripts):
             print('Failed to run dist script {!r}'.format(name))
             sys.exit(1)
 
+def git_root(src_root):
+    # Cannot use --show-toplevel here because git in our CI prints cygwin paths
+    # that python cannot resolve. Workaround this by taking parent of src_root.
+    prefix = quiet_git(['rev-parse', '--show-prefix'], src_root, check=True)[1].strip()
+    if not prefix:
+        return Path(src_root)
+    prefix_level = len(Path(prefix).parents)
+    return Path(src_root).parents[prefix_level - 1]
+
 def is_git(src_root):
-    _git = os.path.join(src_root, '.git')
-    return os.path.isdir(_git) or os.path.isfile(_git)
+    '''
+    Checks if meson.build file at the root source directory is tracked by git.
+    It could be a subproject part of the parent project git repository.
+    '''
+    return quiet_git(['ls-files', '--error-unmatch', 'meson.build'], src_root)[0]
 
 def git_have_dirty_index(src_root):
     '''Check whether there are uncommitted changes in git'''
@@ -110,8 +122,20 @@ def git_clone(src_root, distdir):
         mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
     if os.path.exists(distdir):
         shutil.rmtree(distdir)
-    os.makedirs(distdir)
-    subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
+    repo_root = git_root(src_root)
+    if repo_root.samefile(src_root):
+        os.makedirs(distdir)
+        subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
+    else:
+        subdir = Path(src_root).relative_to(repo_root)
+        tmp_distdir = distdir + '-tmp'
+        if os.path.exists(tmp_distdir):
+            shutil.rmtree(tmp_distdir)
+        os.makedirs(tmp_distdir)
+        subprocess.check_call(['git', 'clone', '--shared', '--no-checkout', str(repo_root), tmp_distdir])
+        subprocess.check_call(['git', 'checkout', 'HEAD', '--', str(subdir)], cwd=tmp_distdir)
+        Path(tmp_distdir, subdir).rename(distdir)
+        shutil.rmtree(tmp_distdir)
     process_submodules(distdir)
     del_gitfiles(distdir)
 

run_unittests.py

@@ -30,7 +30,7 @@ import functools
 import io
 import operator
 import threading
-import zipfile
+import zipfile, tarfile
 import hashlib
 from itertools import chain
 from unittest import mock
@@ -146,6 +146,9 @@ def _git_init(project_dir):
                            'user.name', 'Author Person'], cwd=project_dir)
     subprocess.check_call(['git', 'config',
                            'user.email', 'teh_coderz@example.com'], cwd=project_dir)
+    _git_add_all(project_dir)
+
+def _git_add_all(project_dir):
     subprocess.check_call('git add *', cwd=project_dir, shell=True,
                           stdout=subprocess.DEVNULL)
     subprocess.check_call(['git', 'commit', '-a', '-m', 'I am a project'], cwd=project_dir,
@@ -2846,7 +2849,7 @@ class AllPlatformTests(BasePlatformTests):
             raise unittest.SkipTest('Dist is only supported with Ninja')
 
         try:
-            self.dist_impl(_git_init)
+            self.dist_impl(_git_init, _git_add_all)
         except PermissionError:
             # When run under Windows CI, something (virus scanner?)
             # holds on to the git files so cleaning up the dir
@@ -2915,10 +2918,10 @@ class AllPlatformTests(BasePlatformTests):
         path = os.path.join(project_dir, 'subprojects', name)
         os.makedirs(path)
         with open(os.path.join(path, 'meson.build'), 'w') as ofile:
-            ofile.write("project('{}')".format(name))
+            ofile.write("project('{}', version: '1.0')".format(name))
         return path
 
-    def dist_impl(self, vcs_init, include_subprojects=True):
+    def dist_impl(self, vcs_init, vcs_add_all=None, include_subprojects=True):
         # Create this on the fly because having rogue .git directories inside
         # the source tree leads to all kinds of trouble.
         with tempfile.TemporaryDirectory() as project_dir:
@@ -2929,6 +2932,7 @@ class AllPlatformTests(BasePlatformTests):
                     test('dist test', e)
                     subproject('vcssub', required : false)
                     subproject('tarballsub', required : false)
+                    subproject('samerepo', required : false)
                     '''))
             with open(os.path.join(project_dir, 'distexe.c'), 'w') as ofile:
                 ofile.write(textwrap.dedent('''\
@@ -2948,6 +2952,8 @@ class AllPlatformTests(BasePlatformTests):
                 vcs_init(self.create_dummy_subproject(project_dir, 'vcssub'))
                 self.create_dummy_subproject(project_dir, 'tarballsub')
                 self.create_dummy_subproject(project_dir, 'unusedsub')
+            if vcs_add_all:
+                vcs_add_all(self.create_dummy_subproject(project_dir, 'samerepo'))
             self.init(project_dir)
             self.build('dist')
             self.assertPathExists(xz_distfile)
@@ -2960,24 +2966,46 @@ class AllPlatformTests(BasePlatformTests):
             self.assertPathExists(zip_checksumfile)
 
             if include_subprojects:
+                # Verify that without --include-subprojects we have files from
+                # the main project and also files from subprojects part of the
+                # main vcs repository.
                 z = zipfile.ZipFile(zip_distfile)
-                self.assertEqual(sorted(['disttest-1.4.3/',
-                                         'disttest-1.4.3/meson.build',
-                                         'disttest-1.4.3/distexe.c']),
+                expected = ['disttest-1.4.3/',
+                            'disttest-1.4.3/meson.build',
+                            'disttest-1.4.3/distexe.c']
+                if vcs_add_all:
+                    expected += ['disttest-1.4.3/subprojects/',
+                                 'disttest-1.4.3/subprojects/samerepo/',
+                                 'disttest-1.4.3/subprojects/samerepo/meson.build']
+                self.assertEqual(sorted(expected),
                                  sorted(z.namelist()))
-
+                # Verify that with --include-subprojects we now also have files
+                # from tarball and separate vcs subprojects. But not files from
+                # unused subprojects.
                 self._run(self.meson_command + ['dist', '--formats', 'zip', '--include-subprojects'],
                           workdir=self.builddir)
                 z = zipfile.ZipFile(zip_distfile)
-                self.assertEqual(sorted(['disttest-1.4.3/',
-                                         'disttest-1.4.3/subprojects/',
-                                         'disttest-1.4.3/meson.build',
-                                         'disttest-1.4.3/distexe.c',
-                                         'disttest-1.4.3/subprojects/tarballsub/',
-                                         'disttest-1.4.3/subprojects/vcssub/',
-                                         'disttest-1.4.3/subprojects/tarballsub/meson.build',
-                                         'disttest-1.4.3/subprojects/vcssub/meson.build']),
+                expected += ['disttest-1.4.3/subprojects/tarballsub/',
+                             'disttest-1.4.3/subprojects/tarballsub/meson.build',
+                             'disttest-1.4.3/subprojects/vcssub/',
+                             'disttest-1.4.3/subprojects/vcssub/meson.build']
+                self.assertEqual(sorted(expected),
                                  sorted(z.namelist()))
+            if vcs_add_all:
+                # Verify we can distribute separately subprojects in the same vcs
+                # repository as the main project.
+                subproject_dir = os.path.join(project_dir, 'subprojects', 'samerepo')
+                self.new_builddir()
+                self.init(subproject_dir)
+                self.build('dist')
+                xz_distfile = os.path.join(self.distdir, 'samerepo-1.0.tar.xz')
+                xz_checksumfile = xz_distfile + '.sha256sum'
+                self.assertPathExists(xz_distfile)
+                self.assertPathExists(xz_checksumfile)
+                tar = tarfile.open(xz_distfile, "r:xz")
+                self.assertEqual(sorted(['samerepo-1.0',
+                                         'samerepo-1.0/meson.build']),
+                                 sorted([i.name for i in tar]))
 
     def test_rpath_uses_ORIGIN(self):
         '''