Merge pull request #1516 from centricular/git-submodule-subprojects

Add support for using git submodules as subprojects

mesonbuild/coredata.py

@@ -127,7 +127,7 @@ class CoreData:
             self.cross_file = os.path.join(os.getcwd(), options.cross_file)
         else:
             self.cross_file = None
-
+        self.wrap_mode = options.wrap_mode
         self.compilers = {}
         self.cross_compilers = {}
         self.deps = {}

mesonbuild/interpreter.py

@@ -20,7 +20,7 @@ from . import mlog
 from . import build
 from . import optinterpreter
 from . import compilers
-from .wrap import wrap
+from .wrap import wrap, WrapMode
 from . import mesonlib
 from .mesonlib import FileMode, Popen_safe
 from .dependencies import InternalDependency, Dependency
@@ -1498,11 +1498,13 @@ class Interpreter(InterpreterBase):
             raise InvalidCode('Recursive include of subprojects: %s.' % incpath)
         if dirname in self.subprojects:
             return self.subprojects[dirname]
-        r = wrap.Resolver(os.path.join(self.build.environment.get_source_dir(), self.subproject_dir))
+        subproject_dir_abs = os.path.join(self.environment.get_source_dir(), self.subproject_dir)
-        resolved = r.resolve(dirname)
+        r = wrap.Resolver(subproject_dir_abs, self.coredata.wrap_mode)
-        if resolved is None:
+        try:
-            msg = 'Subproject directory {!r} does not exist and cannot be downloaded.'
+            resolved = r.resolve(dirname)
-            raise InterpreterException(msg.format(os.path.join(self.subproject_dir, dirname)))
+        except RuntimeError as e:
+            msg = 'Subproject directory {!r} does not exist and cannot be downloaded:\n{}'
+            raise InterpreterException(msg.format(os.path.join(self.subproject_dir, dirname), e))
         subdir = os.path.join(self.subproject_dir, resolved)
         os.makedirs(os.path.join(self.build.environment.get_build_dir(), subdir), exist_ok=True)
         self.args_frozen = True
@@ -1909,6 +1911,11 @@ requirements use the version keyword argument instead.''')
         return fbinfo
 
     def dependency_fallback(self, name, kwargs):
+        if self.coredata.wrap_mode in (WrapMode.nofallback, WrapMode.nodownload):
+            mlog.log('Not looking for a fallback subproject for the dependency',
+                     mlog.bold(name), 'because:\nAutomatic wrap-based fallback '
+                     'dependency downloading is disabled.')
+            return None
         dirname, varname = self.get_subproject_infos(kwargs)
         # Try to execute the subproject
         try:

mesonbuild/mesonmain.py

@@ -20,6 +20,7 @@ from . import build
 import platform
 from . import mlog, coredata
 from .mesonlib import MesonException
+from .wrap import WrapMode
 
 
 parser = argparse.ArgumentParser()
@@ -67,6 +68,10 @@ parser.add_argument('-D', action='append', dest='projectoptions', default=[],
                     help='Set project options.')
 parser.add_argument('-v', '--version', action='version',
                     version=coredata.version)
+                    # See the mesonlib.WrapMode enum for documentation
+parser.add_argument('--wrap-mode', default=WrapMode.default,
+                    type=lambda t: getattr(WrapMode, t), choices=WrapMode,
+                    help='Special wrap mode to use')
 parser.add_argument('directories', nargs='*')
 
 class MesonApp:

mesonbuild/wrap/__init__.py

@@ -0,0 +1,31 @@
+from enum import Enum
+
+# Used for the --wrap-mode command-line argument
+#
+# Special wrap modes:
+#   nofallback: Don't download wraps for dependency() fallbacks
+#   nodownload: Don't download wraps for all subproject() calls
+#
+# subprojects are used for two purposes:
+# 1. To download and build dependencies by using .wrap
+#    files if they are not provided by the system. This is
+#    usually expressed via dependency(..., fallback: ...).
+# 2. To download and build 'copylibs' which are meant to be
+#    used by copying into your project. This is always done
+#    with an explicit subproject() call.
+#
+# --wrap-mode=nofallback will never do (1)
+# --wrap-mode=nodownload will do neither (1) nor (2)
+#
+# If you are building from a release tarball, you should be
+# able to safely use 'nodownload' since upstream is
+# expected to ship all required sources with the tarball.
+#
+# If you are building from a git repository, you will want
+# to use 'nofallback' so that any 'copylib' wraps will be
+# download as subprojects.
+#
+# Note that these options do not affect subprojects that
+# are git submodules since those are only usable in git
+# repositories, and you almost always want to download them.
+WrapMode = Enum('WrapMode', 'default nofallback nodownload')

mesonbuild/wrap/wrap.py

@@ -17,6 +17,8 @@ import contextlib
 import urllib.request, os, hashlib, shutil
 import subprocess
 import sys
+from pathlib import Path
+from . import WrapMode
 
 try:
     import ssl
@@ -36,6 +38,13 @@ def build_ssl_context():
     ctx.load_default_certs()
     return ctx
 
+def quiet_git(cmd):
+    pc = subprocess.Popen(['git'] + cmd, stdout=subprocess.PIPE)
+    out, err = pc.communicate()
+    if pc.returncode != 0:
+        return False, err
+    return True, out
+
 def open_wrapdburl(urlstring):
     global ssl_warning_printed
     if has_ssl:
@@ -86,29 +95,45 @@ class PackageDefinition:
         return 'patch_url' in self.values
 
 class Resolver:
-    def __init__(self, subdir_root):
+    def __init__(self, subdir_root, wrap_mode=WrapMode(1)):
+        self.wrap_mode = wrap_mode
         self.subdir_root = subdir_root
         self.cachedir = os.path.join(self.subdir_root, 'packagecache')
 
     def resolve(self, packagename):
-        fname = os.path.join(self.subdir_root, packagename + '.wrap')
+        # Check if the directory is already resolved
-        dirname = os.path.join(self.subdir_root, packagename)
+        dirname = Path(os.path.join(self.subdir_root, packagename))
-        try:
+        subprojdir = os.path.join(*dirname.parts[-2:])
-            if os.listdir(dirname):
+        if dirname.is_dir():
-                # The directory is there and not empty? Great, use it.
+            if (dirname / 'meson.build').is_file():
+                # The directory is there and has meson.build? Great, use it.
                 return packagename
-            else:
+            # Is the dir not empty and also not a git submodule dir that is
-                mlog.warning('Subproject directory %s is empty, possibly because of an unfinished'
+            # not checkout properly? Can't do anything, exception!
-                             'checkout, removing to reclone' % dirname)
+            elif next(dirname.iterdir(), None) and not (dirname / '.git').is_file():
-                os.rmdir(dirname)
+                m = '{!r} is not empty and has no meson.build files'
-        except NotADirectoryError:
+                raise RuntimeError(m.format(subprojdir))
-            raise RuntimeError('%s is not a directory, can not use as subproject.' % dirname)
+        elif dirname.exists():
-        except FileNotFoundError:
+            m = '{!r} already exists and is not a dir; cannot use as subproject'
-            pass
+            raise RuntimeError(m.format(subprojdir))
 
+        dirname = str(dirname)
+        # Check if the subproject is a git submodule
+        if self.resolve_git_submodule(dirname):
+            return packagename
+
+        # Don't download subproject data based on wrap file if requested.
+        # Git submodules are ok (see above)!
+        if self.wrap_mode is WrapMode.nodownload:
+            m = 'Automatic wrap-based subproject downloading is disabled'
+            raise RuntimeError(m)
+
+        # Check if there's a .wrap file for this subproject
+        fname = os.path.join(self.subdir_root, packagename + '.wrap')
         if not os.path.isfile(fname):
             # No wrap file with this name? Give up.
-            return None
+            m = 'No {}.wrap found for {!r}'
+            raise RuntimeError(m.format(packagename, subprojdir))
         p = PackageDefinition(fname)
         if p.type == 'file':
             if not os.path.isdir(self.cachedir):
@@ -120,9 +145,31 @@ class Resolver:
         elif p.type == "hg":
             self.get_hg(p)
         else:
-            raise RuntimeError('Unreachable code.')
+            raise AssertionError('Unreachable code.')
         return p.get('directory')
 
+    def resolve_git_submodule(self, dirname):
+        # Are we in a git repository?
+        ret, out = quiet_git(['rev-parse'])
+        if not ret:
+            return False
+        # Is `dirname` a submodule?
+        ret, out = quiet_git(['submodule', 'status', dirname])
+        if not ret:
+            return False
+        # Submodule has not been added, add it
+        if out.startswith(b'-'):
+            if subprocess.call(['git', 'submodule', 'update', dirname]) != 0:
+                return False
+        # Submodule was added already, but it wasn't populated. Do a checkout.
+        elif out.startswith(b' '):
+            if subprocess.call(['git', 'checkout', '.'], cwd=dirname):
+                return True
+        else:
+            m = 'Unknown git submodule output: {!r}'
+            raise AssertionError(m.format(out))
+        return True
+
     def get_git(self, p):
         checkoutdir = os.path.join(self.subdir_root, p.get('directory'))
         revno = p.get('revision')

run_unittests.py

@@ -48,6 +48,7 @@ def get_fake_options(prefix):
     import argparse
     opts = argparse.Namespace()
     opts.cross_file = None
+    opts.wrap_mode = None
     opts.prefix = prefix
     return opts