Chiebot-Mirror
/
meson
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mpi: Use a factory function

Browse Source 
This makes use of proper ConfigTool and PkgConfig dependencies rather
than one big ExternalDependency that internally creates other
dependencies and then copies their attributes.
pull/6544/head
Dylan Baker 5 years ago
parent f6e50da49a
commit baca2cd07d
2 changed files with 150 additions and 198 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      mesonbuild/dependencies/__init__.py
  2. 344
      mesonbuild/dependencies/mpi.py

4
mesonbuild/dependencies/__init__.py
Unescape View File

@ -22,7 +22,7 @@ from .base import ( # noqa: F401
DependencyFactory) DependencyFactory)
from .dev import ValgrindDependency, gmock_factory, gtest_factory, llvm_factory, zlib_factory from .dev import ValgrindDependency, gmock_factory, gtest_factory, llvm_factory, zlib_factory
from .coarrays import coarray_factory from .coarrays import coarray_factory
from .mpi import MPIDependency from .mpi import mpi_factory
from .scalapack import scalapack_factory from .scalapack import scalapack_factory
from .misc import ( from .misc import (
BlocksDependency, OpenMPDependency, cups_factory, curses_factory, gpgme_factory, BlocksDependency, OpenMPDependency, cups_factory, curses_factory, gpgme_factory,
@ -52,7 +52,7 @@ packages.update({
# per-file # per-file
'coarray': coarray_factory, 'coarray': coarray_factory,
'hdf5': HDF5Dependency, 'hdf5': HDF5Dependency,
'mpi': MPIDependency, 'mpi': mpi_factory,
'scalapack': scalapack_factory, 'scalapack': scalapack_factory,
# From misc: # From misc:

344
mesonbuild/dependencies/mpi.py
Unescape View File

@ -12,111 +12,90 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import functools
import typing as T import typing as T
import os import os
import re import re
import subprocess
from .. import mlog from .base import (DependencyMethods, PkgConfigDependency, factory_methods,
from .. import mesonlib ConfigToolDependency, detect_compiler, ExternalDependency)
from ..mesonlib import split_args, listify
from ..environment import detect_cpu_family from ..environment import detect_cpu_family
from .base import (DependencyException, DependencyMethods, ExternalDependency, ExternalProgram,
PkgConfigDependency)
if T.TYPE_CHECKING:
from .base import DependencyType
from ..compilers import Compiler
from ..environment import Environment, MachineChoice
class MPIDependency(ExternalDependency):
def __init__(self, environment, kwargs: dict): @factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.SYSTEM})
language = kwargs.get('language', 'c') def mpi_factory(env: 'Environment', for_machine: 'MachineChoice',
super().__init__('mpi', environment, kwargs, language=language) kwargs: T.Dict[str, T.Any], methods: T.List[DependencyMethods]) -> T.List['DependencyType']:
kwargs['required'] = False language = kwargs.get('language', 'c')
kwargs['silent'] = True if language not in {'c', 'cpp', 'fortran'}:
self.is_found = False # OpenMPI doesn't work without any other languages
methods = listify(self.methods) return []
env_vars = [] candidates = []
default_wrappers = [] compiler = detect_compiler('mpi', env, for_machine, language)
pkgconfig_files = [] compiler_is_intel = compiler.get_id() in {'intel', 'intel-cl'}
# Only OpenMPI has pkg-config, and it doesn't work with the intel compilers
if DependencyMethods.PKGCONFIG in methods and not compiler_is_intel:
pkg_name = None
if language == 'c': if language == 'c':
cid = environment.detect_c_compiler(self.for_machine).get_id() pkg_name = 'ompi-c'
if cid in ('intel', 'intel-cl'):
env_vars.append('I_MPI_CC')
# IntelMPI doesn't have .pc files
default_wrappers.append('mpiicc')
else:
env_vars.append('MPICC')
pkgconfig_files.append('ompi-c')
default_wrappers.append('mpicc')
elif language == 'cpp': elif language == 'cpp':
cid = environment.detect_cpp_compiler(self.for_machine).get_id() pkg_name = 'ompi-cxx'
if cid in ('intel', 'intel-cl'):
env_vars.append('I_MPI_CXX')
# IntelMPI doesn't have .pc files
default_wrappers.append('mpiicpc')
else:
env_vars.append('MPICXX')
pkgconfig_files.append('ompi-cxx')
default_wrappers += ['mpic++', 'mpicxx', 'mpiCC'] # these are not for intelmpi
elif language == 'fortran': elif language == 'fortran':
cid = environment.detect_fortran_compiler(self.for_machine).get_id() pkg_name = 'ompi-fort'
if cid in ('intel', 'intel-cl'): candidates.append(functools.partial(
env_vars.append('I_MPI_F90') PkgConfigDependency, pkg_name, env, kwargs, language=language))
# IntelMPI doesn't have .pc files
default_wrappers.append('mpiifort') if DependencyMethods.CONFIG_TOOL in methods:
else: nwargs = kwargs.copy()
env_vars += ['MPIFC', 'MPIF90', 'MPIF77']
pkgconfig_files.append('ompi-fort') if compiler_is_intel:
default_wrappers += ['mpifort', 'mpif90', 'mpif77'] if env.machines[for_machine].is_windows():
else: nwargs['version_arg'] = '-v'
raise DependencyException('Language {} is not supported with MPI.'.format(language)) nwargs['returncode_value'] = 3
if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods): if language == 'c':
for pkg in pkgconfig_files: tool_names = [os.environ.get('I_MPI_CC'), 'mpiicc']
pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language) elif language == 'cpp':
if pkgdep.found(): tool_names = [os.environ.get('I_MPI_CXX'), 'mpiicpc']
self.compile_args = pkgdep.get_compile_args() elif language == 'fortran':
self.link_args = pkgdep.get_link_args() tool_names = [os.environ.get('I_MPI_F90'), 'mpiifort']
self.version = pkgdep.get_version()
self.is_found = True cls = IntelMPIConfigToolDependency # type: T.Type[ConfigToolDependency]
self.pcdep = pkgdep else: # OpenMPI, which doesn't work with intel
return #
# We try the environment variables for the tools first, but then
if DependencyMethods.AUTO in methods: # fall back to the hardcoded names
for var in env_vars: if language == 'c':
if var in os.environ: tool_names = [os.environ.get('MPICC'), 'mpicc']
wrappers = [os.environ[var]] elif language == 'cpp':
break tool_names = [os.environ.get('MPICXX'), 'mpic++', 'mpicxx', 'mpiCC']
else: elif language == 'fortran':
# Or search for default wrappers. tool_names = [os.environ.get(e) for e in ['MPIFC', 'MPIF90', 'MPIF77']]
wrappers = default_wrappers tool_names.extend(['mpifort', 'mpif90', 'mpif77'])
for prog in wrappers: cls = OpenMPIConfigToolDependency
# Note: Some use OpenMPI with Intel compilers on Linux
result = self._try_openmpi_wrapper(prog, cid) tool_names = [t for t in tool_names if t] # remove empty environment variables
if result is not None: assert tool_names
self.is_found = True
self.version = result[0] nwargs['tools'] = tool_names
self.compile_args = self._filter_compile_args(result[1]) candidates.append(functools.partial(
self.link_args = self._filter_link_args(result[2], cid) cls, tool_names[0], env, nwargs, language=language))
break
result = self._try_other_wrapper(prog, cid) if DependencyMethods.SYSTEM in methods:
if result is not None: candidates.append(functools.partial(
self.is_found = True MSMPIDependency, 'msmpi', env, kwargs, language=language))
self.version = result[0]
self.compile_args = self._filter_compile_args(result[1]) return candidates
self.link_args = self._filter_link_args(result[2], cid)
break
class _MPIConfigToolDependency(ConfigToolDependency):
if not self.is_found and mesonlib.is_windows():
# only Intel Fortran compiler is compatible with Microsoft MPI at this time.
if language == 'fortran' and cid != 'intel-cl':
return
result = self._try_msmpi()
if result is not None:
self.is_found = True
self.version, self.compile_args, self.link_args = result
return
def _filter_compile_args(self, args: T.Sequence[str]) -> T.List[str]: def _filter_compile_args(self, args: T.Sequence[str]) -> T.List[str]:
""" """
@ -142,7 +121,7 @@ class MPIDependency(ExternalDependency):
result.append(f) result.append(f)
return result return result
def _filter_link_args(self, args: T.Sequence[str], cid: str) -> T.List[str]: def _filter_link_args(self, args: T.Sequence[str]) -> T.List[str]:
""" """
MPI wrappers return a bunch of garbage args. MPI wrappers return a bunch of garbage args.
Drop -O2 and everything that is not needed. Drop -O2 and everything that is not needed.
@ -150,7 +129,7 @@ class MPIDependency(ExternalDependency):
result = [] result = []
include_next = False include_next = False
for f in args: for f in args:
if self._is_link_arg(f, cid): if self._is_link_arg(f):
result.append(f) result.append(f)
if f in ('-L', '-Xlinker'): if f in ('-L', '-Xlinker'):
include_next = True include_next = True
@ -159,121 +138,94 @@ class MPIDependency(ExternalDependency):
result.append(f) result.append(f)
return result return result
@staticmethod def _is_link_arg(self, f: str) -> bool:
def _is_link_arg(f: str, cid: str) -> bool: if self.clib_compiler.id == 'intel-cl':
if cid == 'intel-cl':
return f == '/link' or f.startswith('/LIBPATH') or f.endswith('.lib') # always .lib whether static or dynamic return f == '/link' or f.startswith('/LIBPATH') or f.endswith('.lib') # always .lib whether static or dynamic
else: else:
return (f.startswith(('-L', '-l', '-Xlinker')) or return (f.startswith(('-L', '-l', '-Xlinker')) or
f == '-pthread' or f == '-pthread' or
(f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror'))) (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror')))
def _try_openmpi_wrapper(self, prog, cid: str):
# https://www.open-mpi.org/doc/v4.0/man1/mpifort.1.php class IntelMPIConfigToolDependency(_MPIConfigToolDependency):
if cid == 'intel-cl': # IntelCl doesn't support OpenMPI
return None """Wrapper around Intel's mpiicc and friends."""
prog = ExternalProgram(prog, silent=True)
if not prog.found(): version_arg = '-v' # --version is not the same as -v
return None
def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
# compiler args language: T.Optional[str] = None):
cmd = prog.get_command() + ['--showme:compile'] super().__init__(name, env, kwargs, language=language)
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) if not self.is_found:
if p.returncode != 0: return
mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
mlog.debug(mlog.bold('Standard output\n'), p.stdout) args = self.get_config_value(['-show'], 'link and compile args')
mlog.debug(mlog.bold('Standard error\n'), p.stderr) self.compile_args = self._filter_compile_args(args)
return None self.link_args = self._filter_link_args(args)
cargs = split_args(p.stdout)
# link args def _sanitize_version(self, out: str) -> str:
cmd = prog.get_command() + ['--showme:link'] v = re.search(r'(\d{4}) Update (\d)', out)
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
if p.returncode != 0:
mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
mlog.debug(mlog.bold('Standard output\n'), p.stdout)
mlog.debug(mlog.bold('Standard error\n'), p.stderr)
return None
libs = split_args(p.stdout)
# version
cmd = prog.get_command() + ['--showme:version']
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
if p.returncode != 0:
mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
mlog.debug(mlog.bold('Standard output\n'), p.stdout)
mlog.debug(mlog.bold('Standard error\n'), p.stderr)
return None
v = re.search(r'\d+.\d+.\d+', p.stdout)
if v: if v:
version = v.group(0) return '{}.{}'.format(v.group(1), v.group(2))
else: return out
version = None
return version, cargs, libs
def _try_other_wrapper(self, prog, cid: str) -> T.Tuple[str, T.List[str], T.List[str]]: class OpenMPIConfigToolDependency(_MPIConfigToolDependency):
prog = ExternalProgram(prog, silent=True)
if not prog.found():
return None
cmd = prog.get_command() """Wrapper around OpenMPI mpicc and friends."""
if cid == 'intel-cl':
cmd.append('/show') version_arg = '--showme:version'
else:
cmd.append('-show') def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) language: T.Optional[str] = None):
if p.returncode != 0: super().__init__(name, env, kwargs, language=language)
mlog.debug('Command', mlog.bold(cmd), 'failed to run:') if not self.is_found:
mlog.debug(mlog.bold('Standard output\n'), p.stdout) return
mlog.debug(mlog.bold('Standard error\n'), p.stderr)
return None c_args = self.get_config_value(['--showme:compile'], 'compile_args')
self.compile_args = self._filter_compile_args(c_args)
version = None
stdout = p.stdout l_args = self.get_config_value(['--showme:link'], 'link_args')
if 'Intel(R) MPI Library' in p.stdout: # intel-cl: remove messy compiler logo self.link_args = self._filter_link_args(l_args)
out = stdout.split('\n', 2)
version = out[0] def _sanitize_version(self, out: str) -> str:
stdout = out[2] v = re.search(r'\d+.\d+.\d+', out)
if v:
if version is None: return v.group(0)
p = subprocess.run(cmd + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) return out
if p.returncode == 0:
version = p.stdout.split('\n', 1)[0]
class MSMPIDependency(ExternalDependency):
args = split_args(stdout)
"""The Microsoft MPI."""
return version, args, args
def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any],
def _try_msmpi(self) -> T.Tuple[str, T.List[str], T.List[str]]: language: T.Optional[str] = None):
if self.language == 'cpp': super().__init__(name, env, kwargs, language=language)
# MS-MPI does not support the C++ version of MPI, only the standard C API. # MSMPI only supports the C API
return None if language not in {'c', 'fortran', None}:
if 'MSMPI_INC' not in os.environ: self.is_found = False
return None return
# MSMPI is only for windows, obviously
incdir = os.environ['MSMPI_INC'] if not self.env.machines[self.for_machine].is_windows():
return
incdir = os.environ.get('MSMPI_INC')
arch = detect_cpu_family(self.env.coredata.compilers.host) arch = detect_cpu_family(self.env.coredata.compilers.host)
libdir = None
if arch == 'x86': if arch == 'x86':
if 'MSMPI_LIB32' not in os.environ: libdir = os.environ.get('MSMPI_LIB32')
return None
libdir = os.environ['MSMPI_LIB32']
post = 'x86' post = 'x86'
elif arch == 'x86_64': elif arch == 'x86_64':
if 'MSMPI_LIB64' not in os.environ: libdir = os.environ.get('MSMPI_LIB64')
return None
libdir = os.environ['MSMPI_LIB64']
post = 'x64' post = 'x64'
else:
return None
if self.language == 'fortran': if libdir is None or incdir is None:
return (None, self.is_found = False
['-I' + incdir, '-I' + os.path.join(incdir, post)], return
[os.path.join(libdir, 'msmpi.lib'), os.path.join(libdir, 'msmpifec.lib')])
else:
return (None,
['-I' + incdir, '-I' + os.path.join(incdir, post)],
[os.path.join(libdir, 'msmpi.lib')])
@staticmethod self.is_found = True
def get_methods(): self.link_args = ['-l' + os.path.join(libdir, 'msmpi')]
return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG] self.compile_args = ['-I' + incdir, '-I' + os.path.join(incdir, post)]
if self.language == 'fortran':
self.link_args.append('-l' + os.path.join(libdir, 'msmpifec'))

Write Preview
Loading…
Cancel
Save