diff --git a/.github/workflows/lint_mypy.yml b/.github/workflows/lint_mypy.yml index 235c9cb5c..3ce18596c 100644 --- a/.github/workflows/lint_mypy.yml +++ b/.github/workflows/lint_mypy.yml @@ -30,4 +30,4 @@ jobs: with: python-version: '3.x' - run: python -m pip install mypy - - run: mypy --follow-imports=skip mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py + - run: mypy --follow-imports=skip mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py mesonbuild/dependencies/mpi.py diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index 095ec1d5e..c2c5e6d6e 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -20,7 +20,8 @@ from .base import ( # noqa: F401 PkgConfigDependency, CMakeDependency, find_external_dependency, get_dep_identifier, packages, _packages_accept_language) from .dev import GMockDependency, GTestDependency, LLVMDependency, ValgrindDependency from .coarrays import CoarrayDependency -from .misc import (BlocksDependency, HDF5Dependency, MPIDependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency) +from .mpi import MPIDependency +from .misc import (BlocksDependency, HDF5Dependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency) from .platform import AppleFrameworks from .ui import GLDependency, GnuStepDependency, Qt4Dependency, Qt5Dependency, SDL2Dependency, WxDependency, VulkanDependency diff --git a/mesonbuild/dependencies/misc.py b/mesonbuild/dependencies/misc.py index 56a852e11..789015a83 100644 --- a/mesonbuild/dependencies/misc.py +++ b/mesonbuild/dependencies/misc.py @@ -16,18 +16,16 @@ from pathlib import Path import functools -import os import re import sysconfig from .. import mlog from .. import mesonlib -from ..mesonlib import split_args from ..environment import detect_cpu_family from .base import ( DependencyException, DependencyMethods, ExternalDependency, - ExternalProgram, ExtraFrameworkDependency, PkgConfigDependency, + ExtraFrameworkDependency, PkgConfigDependency, CMakeDependency, ConfigToolDependency, ) @@ -108,206 +106,6 @@ class NetCDFDependency(ExternalDependency): self.is_found = True self.pcdep.append(pkgdep) -class MPIDependency(ExternalDependency): - - def __init__(self, environment, kwargs): - language = kwargs.get('language', 'c') - super().__init__('mpi', environment, language, kwargs) - kwargs['required'] = False - kwargs['silent'] = True - self.is_found = False - - # NOTE: Only OpenMPI supplies a pkg-config file at the moment. - if language == 'c': - env_vars = ['MPICC'] - pkgconfig_files = ['ompi-c'] - default_wrappers = ['mpicc'] - elif language == 'cpp': - env_vars = ['MPICXX'] - pkgconfig_files = ['ompi-cxx'] - default_wrappers = ['mpic++', 'mpicxx', 'mpiCC'] - elif language == 'fortran': - env_vars = ['MPIFC', 'MPIF90', 'MPIF77'] - pkgconfig_files = ['ompi-fort'] - default_wrappers = ['mpifort', 'mpif90', 'mpif77'] - else: - raise DependencyException('Language {} is not supported with MPI.'.format(language)) - - for pkg in pkgconfig_files: - try: - pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language) - if pkgdep.found(): - self.compile_args = pkgdep.get_compile_args() - self.link_args = pkgdep.get_link_args() - self.version = pkgdep.get_version() - self.is_found = True - self.pcdep = pkgdep - break - except Exception: - pass - - if not self.is_found: - # Prefer environment. - for var in env_vars: - if var in os.environ: - wrappers = [os.environ[var]] - break - else: - # Or search for default wrappers. - wrappers = default_wrappers - - for prog in wrappers: - result = self._try_openmpi_wrapper(prog) - if result is not None: - self.is_found = True - self.version = result[0] - self.compile_args = self._filter_compile_args(result[1]) - self.link_args = self._filter_link_args(result[2]) - break - result = self._try_other_wrapper(prog) - if result is not None: - self.is_found = True - self.version = result[0] - self.compile_args = self._filter_compile_args(result[1]) - self.link_args = self._filter_link_args(result[2]) - break - - if not self.is_found and mesonlib.is_windows(): - # only Intel Fortran compiler is compatible with Microsoft MPI at this time. - if language == 'fortran' and environment.detect_fortran_compiler(self.for_machine).name_string() != 'intel-cl': - return - result = self._try_msmpi() - if result is not None: - self.is_found = True - self.version, self.compile_args, self.link_args = result - - def _filter_compile_args(self, args): - """ - MPI wrappers return a bunch of garbage args. - Drop -O2 and everything that is not needed. - """ - result = [] - multi_args = ('-I', ) - if self.language == 'fortran': - fc = self.env.coredata.compilers[self.for_machine]['fortran'] - multi_args += fc.get_module_incdir_args() - - include_next = False - for f in args: - if f.startswith(('-D', '-f') + multi_args) or f == '-pthread' \ - or (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror')): - result.append(f) - if f in multi_args: - # Path is a separate argument. - include_next = True - elif include_next: - include_next = False - result.append(f) - return result - - def _filter_link_args(self, args): - """ - MPI wrappers return a bunch of garbage args. - Drop -O2 and everything that is not needed. - """ - result = [] - include_next = False - for f in args: - if f.startswith(('-L', '-l', '-Xlinker')) or f == '-pthread' \ - or (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror')): - result.append(f) - if f in ('-L', '-Xlinker'): - include_next = True - elif include_next: - include_next = False - result.append(f) - return result - - def _try_openmpi_wrapper(self, prog): - prog = ExternalProgram(prog, silent=True) - if prog.found(): - cmd = prog.get_command() + ['--showme:compile'] - p, o, e = mesonlib.Popen_safe(cmd) - p.wait() - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), o) - mlog.debug(mlog.bold('Standard error\n'), e) - return - cargs = split_args(o) - - cmd = prog.get_command() + ['--showme:link'] - p, o, e = mesonlib.Popen_safe(cmd) - p.wait() - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), o) - mlog.debug(mlog.bold('Standard error\n'), e) - return - libs = split_args(o) - - cmd = prog.get_command() + ['--showme:version'] - p, o, e = mesonlib.Popen_safe(cmd) - p.wait() - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), o) - mlog.debug(mlog.bold('Standard error\n'), e) - return - version = re.search(r'\d+.\d+.\d+', o) - if version: - version = version.group(0) - else: - version = None - - return version, cargs, libs - - def _try_other_wrapper(self, prog): - prog = ExternalProgram(prog, silent=True) - if prog.found(): - cmd = prog.get_command() + ['-show'] - p, o, e = mesonlib.Popen_safe(cmd) - p.wait() - if p.returncode != 0: - mlog.debug('Command', mlog.bold(cmd), 'failed to run:') - mlog.debug(mlog.bold('Standard output\n'), o) - mlog.debug(mlog.bold('Standard error\n'), e) - return - args = split_args(o) - - version = None - - return version, args, args - - def _try_msmpi(self): - if self.language == 'cpp': - # MS-MPI does not support the C++ version of MPI, only the standard C API. - return - if 'MSMPI_INC' not in os.environ: - return - incdir = os.environ['MSMPI_INC'] - arch = detect_cpu_family(self.env.coredata.compilers.host) - if arch == 'x86': - if 'MSMPI_LIB32' not in os.environ: - return - libdir = os.environ['MSMPI_LIB32'] - post = 'x86' - elif arch == 'x86_64': - if 'MSMPI_LIB64' not in os.environ: - return - libdir = os.environ['MSMPI_LIB64'] - post = 'x64' - else: - return - if self.language == 'fortran': - return (None, - ['-I' + incdir, '-I' + os.path.join(incdir, post)], - [os.path.join(libdir, 'msmpi.lib'), os.path.join(libdir, 'msmpifec.lib')]) - else: - return (None, - ['-I' + incdir, '-I' + os.path.join(incdir, post)], - [os.path.join(libdir, 'msmpi.lib')]) - class OpenMPDependency(ExternalDependency): # Map date of specification release (which is the macro value) to a version. diff --git a/mesonbuild/dependencies/mpi.py b/mesonbuild/dependencies/mpi.py new file mode 100644 index 000000000..63460e5c3 --- /dev/null +++ b/mesonbuild/dependencies/mpi.py @@ -0,0 +1,274 @@ +# Copyright 2013-2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing +import os +import re +import subprocess + +from .. import mlog +from .. import mesonlib +from ..mesonlib import split_args +from ..environment import detect_cpu_family +from .base import DependencyException, ExternalDependency, ExternalProgram, PkgConfigDependency + + +class MPIDependency(ExternalDependency): + + def __init__(self, environment, kwargs: dict): + language = kwargs.get('language', 'c') + super().__init__('mpi', environment, language, kwargs) + kwargs['required'] = False + kwargs['silent'] = True + self.is_found = False + env_vars = [] + default_wrappers = [] + pkgconfig_files = [] + if language == 'c': + cid = environment.detect_c_compiler(self.for_machine).get_id() + if cid in ('intel', 'intel-cl'): + env_vars.append('I_MPI_CC') + # IntelMPI doesn't have .pc files + default_wrappers.append('mpiicc') + else: + env_vars.append('MPICC') + pkgconfig_files.append('ompi-c') + default_wrappers.append('mpicc') + elif language == 'cpp': + cid = environment.detect_cpp_compiler(self.for_machine).get_id() + if cid in ('intel', 'intel-cl'): + env_vars.append('I_MPI_CXX') + # IntelMPI doesn't have .pc files + default_wrappers.append('mpiicpc') + else: + env_vars.append('MPICXX') + pkgconfig_files.append('ompi-cxx') + default_wrappers += ['mpic++', 'mpicxx', 'mpiCC'] # these are not for intelmpi + elif language == 'fortran': + cid = environment.detect_fortran_compiler(self.for_machine).get_id() + if cid in ('intel', 'intel-cl'): + env_vars.append('I_MPI_F90') + # IntelMPI doesn't have .pc files + default_wrappers.append('mpiifort') + else: + env_vars += ['MPIFC', 'MPIF90', 'MPIF77'] + pkgconfig_files.append('ompi-fort') + default_wrappers += ['mpifort', 'mpif90', 'mpif77'] + else: + raise DependencyException('Language {} is not supported with MPI.'.format(language)) + + # 1. try pkg-config + for pkg in pkgconfig_files: + try: + pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language) + if pkgdep.found(): + self.compile_args = pkgdep.get_compile_args() + self.link_args = pkgdep.get_link_args() + self.version = pkgdep.get_version() + self.is_found = True + self.pcdep = pkgdep + return + except Exception: + pass + + # 2. Try environment variables + for var in env_vars: + if var in os.environ: + wrappers = [os.environ[var]] + break + else: + # Or search for default wrappers. + wrappers = default_wrappers + + for prog in wrappers: + # Note: Some use OpenMPI with Intel compilers on Linux + result = self._try_openmpi_wrapper(prog, cid) + if result is not None: + self.is_found = True + self.version = result[0] + self.compile_args = self._filter_compile_args(result[1]) + self.link_args = self._filter_link_args(result[2], cid) + break + result = self._try_other_wrapper(prog, cid) + if result is not None: + self.is_found = True + self.version = result[0] + self.compile_args = self._filter_compile_args(result[1]) + self.link_args = self._filter_link_args(result[2], cid) + break + + if not self.is_found and mesonlib.is_windows(): + # only Intel Fortran compiler is compatible with Microsoft MPI at this time. + if language == 'fortran' and cid != 'intel-cl': + return + result = self._try_msmpi() + if result is not None: + self.is_found = True + self.version, self.compile_args, self.link_args = result + + def _filter_compile_args(self, args: typing.Sequence[str]) -> typing.List[str]: + """ + MPI wrappers return a bunch of garbage args. + Drop -O2 and everything that is not needed. + """ + result = [] + multi_args = ('-I', ) + if self.language == 'fortran': + fc = self.env.coredata.compilers[self.for_machine]['fortran'] + multi_args += fc.get_module_incdir_args() + + include_next = False + for f in args: + if f.startswith(('-D', '-f') + multi_args) or f == '-pthread' \ + or (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror')): + result.append(f) + if f in multi_args: + # Path is a separate argument. + include_next = True + elif include_next: + include_next = False + result.append(f) + return result + + def _filter_link_args(self, args: typing.Sequence[str], cid: str) -> typing.List[str]: + """ + MPI wrappers return a bunch of garbage args. + Drop -O2 and everything that is not needed. + """ + result = [] + include_next = False + for f in args: + if self._is_link_arg(f, cid): + result.append(f) + if f in ('-L', '-Xlinker'): + include_next = True + elif include_next: + include_next = False + result.append(f) + return result + + @staticmethod + def _is_link_arg(f: str, cid: str) -> bool: + if cid == 'intel-cl': + return f == '/link' or f.startswith('/LIBPATH') or f.endswith('.lib') # always .lib whether static or dynamic + else: + return (f.startswith(('-L', '-l', '-Xlinker')) or + f == '-pthread' or + (f.startswith('-W') and f != '-Wall' and not f.startswith('-Werror'))) + + def _try_openmpi_wrapper(self, prog, cid: str): + # https://www.open-mpi.org/doc/v4.0/man1/mpifort.1.php + if cid == 'intel-cl': # IntelCl doesn't support OpenMPI + return None + prog = ExternalProgram(prog, silent=True) + if not prog.found(): + return None + + # compiler args + cmd = prog.get_command() + ['--showme:compile'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) + if p.returncode != 0: + mlog.debug('Command', mlog.bold(cmd), 'failed to run:') + mlog.debug(mlog.bold('Standard output\n'), p.stdout) + mlog.debug(mlog.bold('Standard error\n'), p.stderr) + return None + cargs = split_args(p.stdout) + # link args + cmd = prog.get_command() + ['--showme:link'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) + if p.returncode != 0: + mlog.debug('Command', mlog.bold(cmd), 'failed to run:') + mlog.debug(mlog.bold('Standard output\n'), p.stdout) + mlog.debug(mlog.bold('Standard error\n'), p.stderr) + return None + libs = split_args(p.stdout) + # version + cmd = prog.get_command() + ['--showme:version'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) + if p.returncode != 0: + mlog.debug('Command', mlog.bold(cmd), 'failed to run:') + mlog.debug(mlog.bold('Standard output\n'), p.stdout) + mlog.debug(mlog.bold('Standard error\n'), p.stderr) + return None + v = re.search(r'\d+.\d+.\d+', p.stdout) + if v: + version = v.group(0) + else: + version = None + + return version, cargs, libs + + def _try_other_wrapper(self, prog, cid: str) -> typing.Tuple[str, typing.List[str], typing.List[str]]: + prog = ExternalProgram(prog, silent=True) + if not prog.found(): + return None + + cmd = prog.get_command() + if cid == 'intel-cl': + cmd.append('/show') + else: + cmd.append('-show') + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) + if p.returncode != 0: + mlog.debug('Command', mlog.bold(cmd), 'failed to run:') + mlog.debug(mlog.bold('Standard output\n'), p.stdout) + mlog.debug(mlog.bold('Standard error\n'), p.stderr) + return None + + version = None + stdout = p.stdout + if 'Intel(R) MPI Library' in p.stdout: # intel-cl: remove messy compiler logo + out = stdout.split('\n', 2) + version = out[0] + stdout = out[2] + + if version is None: + p = subprocess.run(cmd + ['-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15) + if p.returncode == 0: + version = p.stdout.split('\n', 1)[0] + + args = split_args(stdout) + + return version, args, args + + def _try_msmpi(self) -> typing.Tuple[str, typing.List[str], typing.List[str]]: + if self.language == 'cpp': + # MS-MPI does not support the C++ version of MPI, only the standard C API. + return None + if 'MSMPI_INC' not in os.environ: + return None + + incdir = os.environ['MSMPI_INC'] + arch = detect_cpu_family(self.env.coredata.compilers.host) + if arch == 'x86': + if 'MSMPI_LIB32' not in os.environ: + return None + libdir = os.environ['MSMPI_LIB32'] + post = 'x86' + elif arch == 'x86_64': + if 'MSMPI_LIB64' not in os.environ: + return None + libdir = os.environ['MSMPI_LIB64'] + post = 'x64' + else: + return None + + if self.language == 'fortran': + return (None, + ['-I' + incdir, '-I' + os.path.join(incdir, post)], + [os.path.join(libdir, 'msmpi.lib'), os.path.join(libdir, 'msmpifec.lib')]) + else: + return (None, + ['-I' + incdir, '-I' + os.path.join(incdir, post)], + [os.path.join(libdir, 'msmpi.lib')]) diff --git a/mesonbuild/linkers.py b/mesonbuild/linkers.py index 6c27d6ef5..371220bfe 100644 --- a/mesonbuild/linkers.py +++ b/mesonbuild/linkers.py @@ -636,7 +636,6 @@ class AppleDynamicLinker(PosixDynamicLinkerMixin, DynamicLinker): return args - class GnuDynamicLinker(GnuLikeDynamicLinkerMixin, PosixDynamicLinkerMixin, DynamicLinker): """Representation of GNU ld.bfd and ld.gold.""" diff --git a/mesonbuild/mesonlib.py b/mesonbuild/mesonlib.py index 52d12bffe..361bc2d51 100644 --- a/mesonbuild/mesonlib.py +++ b/mesonbuild/mesonlib.py @@ -770,7 +770,7 @@ if is_windows(): result += (num_backslashes * 2) * '\\' + '"' return result - def split_args(cmd): + def split_args(cmd: typing.Sequence[str]) -> typing.List[str]: result = [] arg = '' num_backslashes = 0