Merge pull request #6544 from dcbaker/mpi-dependency-factory

Use DependencyFactory and ConfigToolDependency for MPI

docs/markdown/Dependencies.md

@@ -411,7 +411,13 @@ not provide them, it will search for the standard wrapper executables,
 `mpic`, `mpicxx`, `mpic++`, `mpifort`, `mpif90`, `mpif77`. If these
 are not in your path, they can be specified by setting the standard
 environment variables `MPICC`, `MPICXX`, `MPIFC`, `MPIF90`, or
-`MPIF77`, during configuration.
+`MPIF77`, during configuration. It will also try to use the Microsoft
+implementation on windows via the `system` method.
+
+`method` may be `auto`, `config-tool`, `pkg-config` or `system`.
+
+*New in 0.54.0* The `config-tool` and `system` method values. Previous
+versions would always try `pkg-config`, then `config-tool`, then `system`.
 
 ## NetCDF
 

mesonbuild/dependencies/__init__.py

@@ -22,7 +22,7 @@ from .base import (  # noqa: F401
     DependencyFactory)
 from .dev import ValgrindDependency, gmock_factory, gtest_factory, llvm_factory, zlib_factory
 from .coarrays import coarray_factory
-from .mpi import MPIDependency
+from .mpi import mpi_factory
 from .scalapack import scalapack_factory
 from .misc import (
     BlocksDependency, OpenMPDependency, cups_factory, curses_factory, gpgme_factory,
@@ -52,7 +52,7 @@ packages.update({
     # per-file
     'coarray': coarray_factory,
     'hdf5': HDF5Dependency,
-    'mpi': MPIDependency,
+    'mpi': mpi_factory,
     'scalapack': scalapack_factory,
 
     # From misc:

mesonbuild/dependencies/base.py

@@ -384,19 +384,32 @@ class NotFoundDependency(Dependency):
 
 class ConfigToolDependency(ExternalDependency):
 
-    """Class representing dependencies found using a config tool."""
+    """Class representing dependencies found using a config tool.
+
+    Takes the following extra keys in kwargs that it uses internally:
+    :tools List[str]: A list of tool names to use
+    :version_arg str: The argument to pass to the tool to get it's version
+    :returncode_value int: The value of the correct returncode
+        Because some tools are stupid and don't return 0
+    """
 
     tools = None
     tool_name = None
+    version_arg = '--version'
     __strip_version = re.compile(r'^[0-9][0-9.]+')
 
     def __init__(self, name, environment, kwargs, language: T.Optional[str] = None):
         super().__init__('config-tool', environment, kwargs, language=language)
         self.name = name
+        # You may want to overwrite the class version in some cases
         self.tools = listify(kwargs.get('tools', self.tools))
+        if not self.tool_name:
+            self.tool_name = self.tools[0]
+        if 'version_arg' in kwargs:
+            self.version_arg = kwargs['version_arg']
 
         req_version = kwargs.get('version', None)
-        tool, version = self.find_config(req_version)
+        tool, version = self.find_config(req_version, kwargs.get('returncode_value', 0))
         self.config = tool
         self.is_found = self.report_config(version, req_version)
         if not self.is_found:
@@ -415,7 +428,7 @@ class ConfigToolDependency(ExternalDependency):
             return m.group(0).rstrip('.')
         return version
 
-    def find_config(self, versions=None):
+    def find_config(self, versions=None, returncode: int = 0):
         """Helper method that searches for config tool binaries in PATH and
         returns the one that best matches the given version requirements.
         """
@@ -444,10 +457,10 @@ class ConfigToolDependency(ExternalDependency):
                     continue
                 tool = potential_bin.get_command()
             try:
-                p, out = Popen_safe(tool + ['--version'])[:2]
+                p, out = Popen_safe(tool + [self.version_arg])[:2]
             except (FileNotFoundError, PermissionError):
                 continue
-            if p.returncode != 0:
+            if p.returncode != returncode:
                 continue
 
             out = self._sanitize_version(out.strip())

mesonbuild/dependencies/mpi.py

@@ -12,111 +12,90 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import typing as T
 import os
 import re
-import subprocess
 
-from .. import mlog
-from .. import mesonlib
-from ..mesonlib import split_args, listify
+from .base import (DependencyMethods, PkgConfigDependency, factory_methods,
+                   ConfigToolDependency, detect_compiler, ExternalDependency)
 from ..environment import detect_cpu_family
-from .base import (DependencyException, DependencyMethods, ExternalDependency, ExternalProgram,
-                   PkgConfigDependency)
 
+if T.TYPE_CHECKING:
+    from .base import DependencyType
+    from ..compilers import Compiler
+    from ..environment import Environment, MachineChoice
 
-class MPIDependency(ExternalDependency):
 
-    def __init__(self, environment, kwargs: dict):
-        language = kwargs.get('language', 'c')
-        super().__init__('mpi', environment, kwargs, language=language)
-        kwargs['required'] = False
-        kwargs['silent'] = True
-        self.is_found = False
-        methods = listify(self.methods)
+@factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.SYSTEM})
+def mpi_factory(env: 'Environment', for_machine: 'MachineChoice',
+                kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List['DependencyType']:
+    language = kwargs.get('language', 'c')
+    if language not in {'c', 'cpp', 'fortran'}:
+        # OpenMPI doesn't work without any other languages
+        return []
 
-        env_vars = []
-        default_wrappers = []
-        pkgconfig_files = []
+    candidates = []
+    compiler = detect_compiler('mpi', env, for_machine, language)
+    compiler_is_intel = compiler.get_id() in {'intel', 'intel-cl'}
+
+    # Only OpenMPI has pkg-config, and it doesn't work with the intel compilers
+    if DependencyMethods.PKGCONFIG in methods and not compiler_is_intel:
+        pkg_name = None
         if language == 'c':
-            cid = environment.detect_c_compiler(self.for_machine).get_id()
-            if cid in ('intel', 'intel-cl'):
-                env_vars.append('I_MPI_CC')
-                # IntelMPI doesn't have .pc files
-                default_wrappers.append('mpiicc')
-            else:
-                env_vars.append('MPICC')
-                pkgconfig_files.append('ompi-c')
-            default_wrappers.append('mpicc')
+            pkg_name = 'ompi-c'
         elif language == 'cpp':
-            cid = environment.detect_cpp_compiler(self.for_machine).get_id()
-            if cid in ('intel', 'intel-cl'):
-                env_vars.append('I_MPI_CXX')
-                # IntelMPI doesn't have .pc files
-                default_wrappers.append('mpiicpc')
-            else:
-                env_vars.append('MPICXX')
-                pkgconfig_files.append('ompi-cxx')
-                default_wrappers += ['mpic++', 'mpicxx', 'mpiCC']  # these are not for intelmpi
+            pkg_name = 'ompi-cxx'
         elif language == 'fortran':
-            cid = environment.detect_fortran_compiler(self.for_machine).get_id()
-            if cid in ('intel', 'intel-cl'):
-                env_vars.append('I_MPI_F90')
-                # IntelMPI doesn't have .pc files
-                default_wrappers.append('mpiifort')
-            else:
-                env_vars += ['MPIFC', 'MPIF90', 'MPIF77']
-                pkgconfig_files.append('ompi-fort')
-            default_wrappers += ['mpifort', 'mpif90', 'mpif77']
-        else:
-            raise DependencyException('Language {} is not supported with MPI.'.format(language))
-
-        if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
-            for pkg in pkgconfig_files:
-                pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
-                if pkgdep.found():
-                    self.compile_args = pkgdep.get_compile_args()
-                    self.link_args = pkgdep.get_link_args()
-                    self.version = pkgdep.get_version()
-                    self.is_found = True
-                    self.pcdep = pkgdep
-                    return
-
-        if DependencyMethods.AUTO in methods:
-            for var in env_vars:
-                if var in os.environ:
-                    wrappers = [os.environ[var]]
-                    break
-            else:
-                # Or search for default wrappers.
-                wrappers = default_wrappers
-
-            for prog in wrappers:
-                # Note: Some use OpenMPI with Intel compilers on Linux
-                result = self._try_openmpi_wrapper(prog, cid)
-                if result is not None:
-                    self.is_found = True
-                    self.version = result[0]
-                    self.compile_args = self._filter_compile_args(result[1])
-                    self.link_args = self._filter_link_args(result[2], cid)
-                    break
-                result = self._try_other_wrapper(prog, cid)
-                if result is not None:
-                    self.is_found = True
-                    self.version = result[0]
-                    self.compile_args = self._filter_compile_args(result[1])
-                    self.link_args = self._filter_link_args(result[2], cid)
-                    break
-
-            if not self.is_found and mesonlib.is_windows():
-                # only Intel Fortran compiler is compatible with Microsoft MPI at this time.
-                if language == 'fortran' and cid != 'intel-cl':
-                    return
-                result = self._try_msmpi()
-                if result is not None:
-                    self.is_found = True
-                    self.version, self.compile_args, self.link_args = result
-            return
+            pkg_name = 'ompi-fort'
+        candidates.append(functools.partial(
+            PkgConfigDependency, pkg_name, env, kwargs, language=language))
+
+    if DependencyMethods.CONFIG_TOOL in methods:
+        nwargs = kwargs.copy()
+
+        if compiler_is_intel:
+            if env.machines[for_machine].is_windows():
+                nwargs['version_arg'] = '-v'
+                nwargs['returncode_value'] = 3
+
+            if language == 'c':
+                tool_names = [os.environ.get('I_MPI_CC'), 'mpiicc']
+            elif language == 'cpp':
+                tool_names = [os.environ.get('I_MPI_CXX'), 'mpiicpc']
+            elif language == 'fortran':
+                tool_names = [os.environ.get('I_MPI_F90'), 'mpiifort']
+
+            cls = IntelMPIConfigToolDependency  # type: T.Type[ConfigToolDependency]
+        else: # OpenMPI, which doesn't work with intel
+            #
+            # We try the environment variables for the tools first, but then
+            # fall back to the hardcoded names
+            if language == 'c':
+                tool_names = [os.environ.get('MPICC'), 'mpicc']
+            elif language == 'cpp':
+                tool_names = [os.environ.get('MPICXX'), 'mpic++', 'mpicxx', 'mpiCC']
+            elif language == 'fortran':
+                tool_names = [os.environ.get(e) for e in ['MPIFC', 'MPIF90', 'MPIF77']]
+                tool_names.extend(['mpifort', 'mpif90', 'mpif77'])
+
+            cls = OpenMPIConfigToolDependency
+
+        tool_names = [t for t in tool_names if t]  # remove empty environment variables
+        assert tool_names
+
+        nwargs['tools'] = tool_names
+        candidates.append(functools.partial(
+            cls, tool_names[0], env, nwargs, language=language))
+
+    if DependencyMethods.SYSTEM in methods:
+        candidates.append(functools.partial(
+            MSMPIDependency, 'msmpi', env, kwargs, language=language))
+
+    return candidates
+
+
+class _MPIConfigToolDependency(ConfigToolDependency):
 
     def _filter_compile_args(self, args: T.Sequence[str]) -> T.List[str]:
         """
@@ -142,7 +121,7 @@ class MPIDependency(ExternalDependency):
                 result.append(f)
         return result
 
-    def _filter_link_args(self, args: T.Sequence[str], cid: str) -> T.List[str]:
+    def _filter_link_args(self, args: T.Sequence[str]) -> T.List[str]:
         """
         MPI wrappers return a bunch of garbage args.
         Drop -O2 and everything that is not needed.
@@ -150,7 +129,7 @@ class MPIDependency(ExternalDependency):
         result = []
         include_next = False
         for f in args:
-            if self._is_link_arg(f, cid):
+            if self._is_link_arg(f):
                 result.append(f)
                 if f in ('-L', '-Xlinker'):
                     include_next = True
@@ -159,121 +138,94 @@ class MPIDependency(ExternalDependency):
                 result.append(f)
         return result
 
-    @staticmethod
-    def _is_link_arg(f: str, cid: str) -> bool:
-        if cid == 'intel-cl':
+    def _is_link_arg(self, f: str) -> bool:
+        if self.clib_compiler.id == 'intel-cl':
             return f == '/link' or f.startswith('/LIBPATH') or f.endswith('.lib')   # always .lib whether static or dynamic
         else:
             return (f.startswith(('-L', '-l', '-Xlinker')) or
                     f == '-pthread' or
                     (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror')))
 
-    def _try_openmpi_wrapper(self, prog, cid: str):
-        # https://www.open-mpi.org/doc/v4.0/man1/mpifort.1.php
-        if cid == 'intel-cl':  # IntelCl doesn't support OpenMPI
-            return None
-        prog = ExternalProgram(prog, silent=True)
-        if not prog.found():
-            return None
-
-        # compiler args
-        cmd = prog.get_command() + ['--showme:compile']
-        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-        if p.returncode != 0:
-            mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
-            mlog.debug(mlog.bold('Standard output\n'), p.stdout)
-            mlog.debug(mlog.bold('Standard error\n'), p.stderr)
-            return None
-        cargs = split_args(p.stdout)
-        # link args
-        cmd = prog.get_command() + ['--showme:link']
-        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-        if p.returncode != 0:
-            mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
-            mlog.debug(mlog.bold('Standard output\n'), p.stdout)
-            mlog.debug(mlog.bold('Standard error\n'), p.stderr)
-            return None
-        libs = split_args(p.stdout)
-        # version
-        cmd = prog.get_command() + ['--showme:version']
-        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-        if p.returncode != 0:
-            mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
-            mlog.debug(mlog.bold('Standard output\n'), p.stdout)
-            mlog.debug(mlog.bold('Standard error\n'), p.stderr)
-            return None
-        v = re.search(r'\d+.\d+.\d+', p.stdout)
+
+class IntelMPIConfigToolDependency(_MPIConfigToolDependency):
+
+    """Wrapper around Intel's mpiicc and friends."""
+
+    version_arg = '-v'  # --version is not the same as -v
+
+    def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
+                 language: T.Optional[str] = None):
+        super().__init__(name, env, kwargs, language=language)
+        if not self.is_found:
+            return
+
+        args = self.get_config_value(['-show'], 'link and compile args')
+        self.compile_args = self._filter_compile_args(args)
+        self.link_args = self._filter_link_args(args)
+
+    def _sanitize_version(self, out: str) -> str:
+        v = re.search(r'(\d{4}) Update (\d)', out)
         if v:
-            version = v.group(0)
-        else:
-            version = None
+            return '{}.{}'.format(v.group(1), v.group(2))
+        return out
 
-        return version, cargs, libs
 
-    def _try_other_wrapper(self, prog, cid: str) -> T.Tuple[str, T.List[str], T.List[str]]:
-        prog = ExternalProgram(prog, silent=True)
-        if not prog.found():
-            return None
+class OpenMPIConfigToolDependency(_MPIConfigToolDependency):
 
-        cmd = prog.get_command()
-        if cid == 'intel-cl':
-            cmd.append('/show')
-        else:
-            cmd.append('-show')
-        p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-        if p.returncode != 0:
-            mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
-            mlog.debug(mlog.bold('Standard output\n'), p.stdout)
-            mlog.debug(mlog.bold('Standard error\n'), p.stderr)
-            return None
-
-        version = None
-        stdout = p.stdout
-        if 'Intel(R) MPI Library' in p.stdout:  # intel-cl: remove messy compiler logo
-            out = stdout.split('\n', 2)
-            version = out[0]
-            stdout = out[2]
-
-        if version is None:
-            p = subprocess.run(cmd + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-            if p.returncode == 0:
-                version = p.stdout.split('\n', 1)[0]
-
-        args = split_args(stdout)
-
-        return version, args, args
-
-    def _try_msmpi(self) -> T.Tuple[str, T.List[str], T.List[str]]:
-        if self.language == 'cpp':
-            # MS-MPI does not support the C++ version of MPI, only the standard C API.
-            return None
-        if 'MSMPI_INC' not in os.environ:
-            return None
-
-        incdir = os.environ['MSMPI_INC']
+    """Wrapper around OpenMPI mpicc and friends."""
+
+    version_arg = '--showme:version'
+
+    def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
+                 language: T.Optional[str] = None):
+        super().__init__(name, env, kwargs, language=language)
+        if not self.is_found:
+            return
+
+        c_args = self.get_config_value(['--showme:compile'], 'compile_args')
+        self.compile_args = self._filter_compile_args(c_args)
+
+        l_args = self.get_config_value(['--showme:link'], 'link_args')
+        self.link_args = self._filter_link_args(l_args)
+
+    def _sanitize_version(self, out: str) -> str:
+        v = re.search(r'\d+.\d+.\d+', out)
+        if v:
+            return v.group(0)
+        return out
+
+
+class MSMPIDependency(ExternalDependency):
+
+    """The Microsoft MPI."""
+
+    def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
+                 language: T.Optional[str] = None):
+        super().__init__(name, env, kwargs, language=language)
+        # MSMPI only supports the C API
+        if language not in {'c', 'fortran', None}:
+            self.is_found = False
+            return
+        # MSMPI is only for windows, obviously
+        if not self.env.machines[self.for_machine].is_windows():
+            return
+
+        incdir = os.environ.get('MSMPI_INC')
         arch = detect_cpu_family(self.env.coredata.compilers.host)
+        libdir = None
         if arch == 'x86':
-            if 'MSMPI_LIB32' not in os.environ:
-                return None
-            libdir = os.environ['MSMPI_LIB32']
+            libdir = os.environ.get('MSMPI_LIB32')
             post = 'x86'
         elif arch == 'x86_64':
-            if 'MSMPI_LIB64' not in os.environ:
-                return None
-            libdir = os.environ['MSMPI_LIB64']
+            libdir = os.environ.get('MSMPI_LIB64')
             post = 'x64'
-        else:
-            return None
 
-        if self.language == 'fortran':
-            return (None,
-                    ['-I' + incdir, '-I' + os.path.join(incdir, post)],
-                    [os.path.join(libdir, 'msmpi.lib'), os.path.join(libdir, 'msmpifec.lib')])
-        else:
-            return (None,
-                    ['-I' + incdir, '-I' + os.path.join(incdir, post)],
-                    [os.path.join(libdir, 'msmpi.lib')])
+        if libdir is None or incdir is None:
+            self.is_found = False
+            return
 
-    @staticmethod
-    def get_methods():
-        return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]
+        self.is_found = True
+        self.link_args = ['-l' + os.path.join(libdir, 'msmpi')]
+        self.compile_args = ['-I' + incdir, '-I' + os.path.join(incdir, post)]
+        if self.language == 'fortran':
+            self.link_args.append('-l' + os.path.join(libdir, 'msmpifec'))

mesonbuild/dependencies/ui.py

@@ -79,13 +79,13 @@ class GnuStepDependency(ConfigToolDependency):
             ['--gui-libs' if 'gui' in self.modules else '--base-libs'],
             'link_args'))
 
-    def find_config(self, versions=None):
+    def find_config(self, versions=None, returncode: int = 0):
         tool = [self.tools[0]]
         try:
             p, out = Popen_safe(tool + ['--help'])[:2]
         except (FileNotFoundError, PermissionError):
             return (None, None)
-        if p.returncode != 0:
+        if p.returncode != returncode:
             return (None, None)
         self.config = tool
         found_version = self.detect_version()

test cases/frameworks/17 mpi/meson.build

@@ -1,7 +1,9 @@
 project('mpi', 'c', 'cpp', default_options: ['b_asneeded=false'])
 
+method = get_option('method')
+
 cc = meson.get_compiler('c')
-mpic = dependency('mpi', language : 'c', required : false)
+mpic = dependency('mpi', language : 'c', required : false, method : method)
 if not mpic.found()
   error('MESON_SKIP_TEST: MPI not found, skipping.')
 endif
@@ -14,7 +16,7 @@ test('MPI C', exec, timeout: 10)
 
 # C++ MPI not supported by MS-MPI
 cpp = meson.get_compiler('cpp')
-mpicpp = dependency('mpi', language : 'cpp', required: false)
+mpicpp = dependency('mpi', language : 'cpp', required: false, method : method)
 if not cpp.links('''
 #include <mpi.h>
 #include <stdio.h>
@@ -31,7 +33,7 @@ test('MPI C++', execpp, timeout: 10)
 
 if add_languages('fortran', required : false)
   fc = meson.get_compiler('fortran')
-  mpif = dependency('mpi', language : 'fortran', required: false)
+  mpif = dependency('mpi', language : 'fortran', required: false, method : method)
   if not fc.links('use mpi; end', dependencies: mpif, name: 'Fortran MPI')
     mpif = disabler()
   endif
@@ -46,5 +48,5 @@ endif
 
 # Check we can apply a version constraint
 if mpic.version() != 'unknown'
-  dependency('mpi', version: '>=@0@'.format(mpic.version()))
+  dependency('mpi', version: '>=@0@'.format(mpic.version()), method : method)
 endif

test cases/frameworks/17 mpi/meson_options.txt

@@ -0,0 +1,6 @@
+option(
+    'method',
+    type : 'combo',
+    choices : ['auto', 'pkg-config', 'config-tool', 'system'],
+    value : 'auto',
+)