Merge pull request #6355 from scivision/depmethod

dependencies: refactor {coarray,mpi,hdf5,netcdf} to use dependency(.., methods: ...)

mesonbuild/dependencies/coarrays.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .base import CMakeDependency, ExternalDependency, PkgConfigDependency
+from ..mesonlib import listify
+from .base import CMakeDependency, DependencyMethods, ExternalDependency, PkgConfigDependency
 
 
 class CoarrayDependency(ExternalDependency):
@@ -29,33 +30,41 @@ class CoarrayDependency(ExternalDependency):
         kwargs['required'] = False
         kwargs['silent'] = True
         self.is_found = False
+        methods = listify(self.methods)
 
         cid = self.get_compiler().get_id()
         if cid == 'gcc':
             """ OpenCoarrays is the most commonly used method for Fortran Coarray with GCC """
-            # first try pkg-config
-            for pkg in ['caf-openmpi', 'caf']:
-                pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
-                if pkgdep.found():
-                    self.compile_args = pkgdep.get_compile_args()
-                    self.link_args = pkgdep.get_link_args()
-                    self.version = pkgdep.get_version()
+
+            if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
+                for pkg in ['caf-openmpi', 'caf']:
+                    pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
+                    if pkgdep.found():
+                        self.compile_args = pkgdep.get_compile_args()
+                        self.link_args = pkgdep.get_link_args()
+                        self.version = pkgdep.get_version()
+                        self.is_found = True
+                        self.pcdep = pkgdep
+                        return
+
+            if set([DependencyMethods.AUTO, DependencyMethods.CMAKE]).intersection(methods):
+                if not kwargs.get('modules'):
+                    kwargs['modules'] = 'OpenCoarrays::caf_mpi'
+                cmakedep = CMakeDependency('OpenCoarrays', environment, kwargs, language=self.language)
+                if cmakedep.found():
+                    self.compile_args = cmakedep.get_compile_args()
+                    self.link_args = cmakedep.get_link_args()
+                    self.version = cmakedep.get_version()
                     self.is_found = True
-                    self.pcdep = pkgdep
                     return
-            # second try CMake
-            kwargs['modules'] = 'OpenCoarrays::caf_mpi'
-            cmakedep = CMakeDependency('OpenCoarrays', environment, kwargs)
-            if cmakedep.found():
-                self.compile_args = cmakedep.get_compile_args()
-                self.link_args = cmakedep.get_link_args()
-                self.version = cmakedep.get_version()
+
+            if DependencyMethods.AUTO in methods:
+                # fallback to single image
+                self.compile_args = ['-fcoarray=single']
+                self.version = 'single image (fallback)'
                 self.is_found = True
                 return
-            # give up, just run as single image fallback
-            self.compile_args = ['-fcoarray=single']
-            self.version = 'single image'
-            self.is_found = True
+
         elif cid == 'intel':
             """ Coarrays are built into Intel compilers, no external library needed """
             self.is_found = True
@@ -68,3 +77,7 @@ class CoarrayDependency(ExternalDependency):
         elif cid == 'nagfor':
             """ NAG doesn't require any special arguments for Coarray """
             self.is_found = True
+
+    @staticmethod
+    def get_methods():
+        return [DependencyMethods.AUTO, DependencyMethods.CMAKE, DependencyMethods.PKGCONFIG]

mesonbuild/dependencies/hdf5.py

@@ -15,11 +15,13 @@
 # This file contains the detection logic for miscellaneous external dependencies.
 
 import subprocess
+import shutil
 from pathlib import Path
 
 from .. import mlog
-from ..mesonlib import split_args
-from .base import DependencyException, ExternalDependency, ExternalProgram, PkgConfigDependency
+from ..mesonlib import split_args, listify
+from .base import (DependencyException, DependencyMethods, ExternalDependency, ExternalProgram,
+                   PkgConfigDependency)
 
 class HDF5Dependency(ExternalDependency):
 
@@ -29,95 +31,100 @@ class HDF5Dependency(ExternalDependency):
         kwargs['required'] = False
         kwargs['silent'] = True
         self.is_found = False
+        methods = listify(self.methods)
 
-        # 1. pkg-config
-        pkgconfig_files = ['hdf5', 'hdf5-serial']
-        # some distros put hdf5-1.2.3.pc with version number in .pc filename.
-        try:
-            ret = subprocess.run(['pkg-config', '--list-all'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
-                                 universal_newlines=True)
-            if ret.returncode == 0:
-                for pkg in ret.stdout.split('\n'):
-                    if pkg.startswith(('hdf5')):
-                        pkgconfig_files.append(pkg.split(' ', 1)[0])
-                pkgconfig_files = list(set(pkgconfig_files))  # dedupe
-        except FileNotFoundError:
-            # pkg-config was not available
-            pass
         if language not in ('c', 'cpp', 'fortran'):
             raise DependencyException('Language {} is not supported with HDF5.'.format(language))
 
-        for pkg in pkgconfig_files:
-            pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
-            if not pkgdep.found():
-                continue
-
-            self.compile_args = pkgdep.get_compile_args()
-            # some broken pkgconfig don't actually list the full path to the needed includes
-            newinc = []
-            for arg in self.compile_args:
-                if arg.startswith('-I'):
-                    stem = 'static' if kwargs.get('static', False) else 'shared'
-                    if (Path(arg[2:]) / stem).is_dir():
-                        newinc.append('-I' + str(Path(arg[2:]) / stem))
-            self.compile_args += newinc
-
-            # derive needed libraries by language
-            pd_link_args = pkgdep.get_link_args()
-            link_args = []
-            for larg in pd_link_args:
-                lpath = Path(larg)
-                # some pkg-config hdf5.pc (e.g. Ubuntu) don't include the commonly-used HL HDF5 libraries,
-                # so let's add them if they exist
-                # additionally, some pkgconfig HDF5 HL files are malformed so let's be sure to find HL anyway
-                if lpath.is_file():
-                    hl = []
-                    if language == 'cpp':
-                        hl += ['_hl_cpp', '_cpp']
-                    elif language == 'fortran':
-                        hl += ['_hl_fortran', 'hl_fortran', '_fortran']
-                    hl += ['_hl']  # C HL library, always needed
-
-                    suffix = '.' + lpath.name.split('.', 1)[1]  # in case of .dll.a
-                    for h in hl:
-                        hlfn = lpath.parent / (lpath.name.split('.', 1)[0] + h + suffix)
-                        if hlfn.is_file():
-                            link_args.append(str(hlfn))
-                    # HDF5 C libs are required by other HDF5 languages
-                    link_args.append(larg)
-                else:
-                    link_args.append(larg)
+        if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
+            PCEXE = shutil.which('pkg-config')
+            if PCEXE:
+                pkgconfig_files = ['hdf5', 'hdf5-serial']
+                # some distros put hdf5-1.2.3.pc with version number in .pc filename.
+                ret = subprocess.run([PCEXE, '--list-all'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                                     universal_newlines=True)
+                if ret.returncode == 0:
+                    for pkg in ret.stdout.split('\n'):
+                        if pkg.startswith(('hdf5')):
+                            pkgconfig_files.append(pkg.split(' ', 1)[0])
+                    pkgconfig_files = list(set(pkgconfig_files))  # dedupe
+
+            for pkg in pkgconfig_files:
+                pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
+                if not pkgdep.found():
+                    continue
+
+                self.compile_args = pkgdep.get_compile_args()
+                # some broken pkgconfig don't actually list the full path to the needed includes
+                newinc = []
+                for arg in self.compile_args:
+                    if arg.startswith('-I'):
+                        stem = 'static' if kwargs.get('static', False) else 'shared'
+                        if (Path(arg[2:]) / stem).is_dir():
+                            newinc.append('-I' + str(Path(arg[2:]) / stem))
+                self.compile_args += newinc
+
+                # derive needed libraries by language
+                pd_link_args = pkgdep.get_link_args()
+                link_args = []
+                for larg in pd_link_args:
+                    lpath = Path(larg)
+                    # some pkg-config hdf5.pc (e.g. Ubuntu) don't include the commonly-used HL HDF5 libraries,
+                    # so let's add them if they exist
+                    # additionally, some pkgconfig HDF5 HL files are malformed so let's be sure to find HL anyway
+                    if lpath.is_file():
+                        hl = []
+                        if language == 'cpp':
+                            hl += ['_hl_cpp', '_cpp']
+                        elif language == 'fortran':
+                            hl += ['_hl_fortran', 'hl_fortran', '_fortran']
+                        hl += ['_hl']  # C HL library, always needed
+
+                        suffix = '.' + lpath.name.split('.', 1)[1]  # in case of .dll.a
+                        for h in hl:
+                            hlfn = lpath.parent / (lpath.name.split('.', 1)[0] + h + suffix)
+                            if hlfn.is_file():
+                                link_args.append(str(hlfn))
+                        # HDF5 C libs are required by other HDF5 languages
+                        link_args.append(larg)
+                    else:
+                        link_args.append(larg)
 
+                self.link_args = link_args
+                self.version = pkgdep.get_version()
+                self.is_found = True
+                self.pcdep = pkgdep
+                return
+
+        if DependencyMethods.AUTO in methods:
+            wrappers = {'c': 'h5cc', 'cpp': 'h5c++', 'fortran': 'h5fc'}
+            comp_args = []
+            link_args = []
+            # have to always do C as well as desired language
+            for lang in set([language, 'c']):
+                prog = ExternalProgram(wrappers[lang], silent=True)
+                if not prog.found():
+                    return
+                cmd = prog.get_command() + ['-show']
+                p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
+                if p.returncode != 0:
+                    mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
+                    mlog.debug(mlog.bold('Standard output\n'), p.stdout)
+                    mlog.debug(mlog.bold('Standard error\n'), p.stderr)
+                    return
+                args = split_args(p.stdout)
+                for arg in args[1:]:
+                    if arg.startswith(('-I', '-f', '-D')) or arg == '-pthread':
+                        comp_args.append(arg)
+                    elif arg.startswith(('-L', '-l', '-Wl')):
+                        link_args.append(arg)
+                    elif Path(arg).is_file():
+                        link_args.append(arg)
+            self.compile_args = comp_args
             self.link_args = link_args
-            self.version = pkgdep.get_version()
             self.is_found = True
-            self.pcdep = pkgdep
             return
 
-        # 2. compiler wrapper fallback
-        wrappers = {'c': 'h5cc', 'cpp': 'h5c++', 'fortran': 'h5fc'}
-        comp_args = []
-        link_args = []
-        # have to always do C as well as desired language
-        for lang in set([language, 'c']):
-            prog = ExternalProgram(wrappers[lang], silent=True)
-            if not prog.found():
-                return
-            cmd = prog.get_command() + ['-show']
-            p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
-            if p.returncode != 0:
-                mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
-                mlog.debug(mlog.bold('Standard output\n'), p.stdout)
-                mlog.debug(mlog.bold('Standard error\n'), p.stderr)
-                return
-            args = split_args(p.stdout)
-            for arg in args[1:]:
-                if arg.startswith(('-I', '-f', '-D')) or arg == '-pthread':
-                    comp_args.append(arg)
-                elif arg.startswith(('-L', '-l', '-Wl')):
-                    link_args.append(arg)
-                elif Path(arg).is_file():
-                    link_args.append(arg)
-        self.compile_args = comp_args
-        self.link_args = link_args
-        self.is_found = True
+    @staticmethod
+    def get_methods():
+        return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]

mesonbuild/dependencies/misc.py

@@ -39,26 +39,41 @@ class NetCDFDependency(ExternalDependency):
         kwargs['required'] = False
         kwargs['silent'] = True
         self.is_found = False
-
-        pkgconfig_files = ['netcdf']
+        methods = listify(self.methods)
 
         if language not in ('c', 'cpp', 'fortran'):
             raise DependencyException('Language {} is not supported with NetCDF.'.format(language))
 
-        if language == 'fortran':
-            pkgconfig_files.append('netcdf-fortran')
-
-        self.compile_args = []
-        self.link_args = []
-        self.pcdep = []
-        for pkg in pkgconfig_files:
-            pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
-            if pkgdep.found():
-                self.compile_args.extend(pkgdep.get_compile_args())
-                self.link_args.extend(pkgdep.get_link_args())
-                self.version = pkgdep.get_version()
+        if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
+            pkgconfig_files = ['netcdf']
+
+            if language == 'fortran':
+                pkgconfig_files.append('netcdf-fortran')
+
+            self.compile_args = []
+            self.link_args = []
+            self.pcdep = []
+            for pkg in pkgconfig_files:
+                pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
+                if pkgdep.found():
+                    self.compile_args.extend(pkgdep.get_compile_args())
+                    self.link_args.extend(pkgdep.get_link_args())
+                    self.version = pkgdep.get_version()
+                    self.is_found = True
+                    self.pcdep.append(pkgdep)
+
+        if set([DependencyMethods.AUTO, DependencyMethods.CMAKE]).intersection(methods):
+            cmakedep = CMakeDependency('NetCDF', environment, kwargs, language=self.language)
+            if cmakedep.found():
+                self.compile_args = cmakedep.get_compile_args()
+                self.link_args = cmakedep.get_link_args()
+                self.version = cmakedep.get_version()
                 self.is_found = True
-                self.pcdep.append(pkgdep)
+                return
+
+    @staticmethod
+    def get_methods():
+        return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG, DependencyMethods.CMAKE]
 
 
 class OpenMPDependency(ExternalDependency):

mesonbuild/dependencies/mpi.py

@@ -19,9 +19,10 @@ import subprocess
 
 from .. import mlog
 from .. import mesonlib
-from ..mesonlib import split_args
+from ..mesonlib import split_args, listify
 from ..environment import detect_cpu_family
-from .base import DependencyException, ExternalDependency, ExternalProgram, PkgConfigDependency
+from .base import (DependencyException, DependencyMethods, ExternalDependency, ExternalProgram,
+                   PkgConfigDependency)
 
 
 class MPIDependency(ExternalDependency):
@@ -32,6 +33,8 @@ class MPIDependency(ExternalDependency):
         kwargs['required'] = False
         kwargs['silent'] = True
         self.is_found = False
+        methods = listify(self.methods)
+
         env_vars = []
         default_wrappers = []
         pkgconfig_files = []
@@ -68,9 +71,8 @@ class MPIDependency(ExternalDependency):
         else:
             raise DependencyException('Language {} is not supported with MPI.'.format(language))
 
-        # 1. try pkg-config
-        for pkg in pkgconfig_files:
-            try:
+        if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
+            for pkg in pkgconfig_files:
                 pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
                 if pkgdep.found():
                     self.compile_args = pkgdep.get_compile_args()
@@ -79,43 +81,42 @@ class MPIDependency(ExternalDependency):
                     self.is_found = True
                     self.pcdep = pkgdep
                     return
-            except Exception:
-                pass
 
-        # 2. Try environment variables
-        for var in env_vars:
-            if var in os.environ:
-                wrappers = [os.environ[var]]
-                break
-        else:
-            # Or search for default wrappers.
-            wrappers = default_wrappers
+        if DependencyMethods.AUTO in methods:
+            for var in env_vars:
+                if var in os.environ:
+                    wrappers = [os.environ[var]]
+                    break
+            else:
+                # Or search for default wrappers.
+                wrappers = default_wrappers
 
-        for prog in wrappers:
-            # Note: Some use OpenMPI with Intel compilers on Linux
-            result = self._try_openmpi_wrapper(prog, cid)
-            if result is not None:
-                self.is_found = True
-                self.version = result[0]
-                self.compile_args = self._filter_compile_args(result[1])
-                self.link_args = self._filter_link_args(result[2], cid)
-                break
-            result = self._try_other_wrapper(prog, cid)
-            if result is not None:
-                self.is_found = True
-                self.version = result[0]
-                self.compile_args = self._filter_compile_args(result[1])
-                self.link_args = self._filter_link_args(result[2], cid)
-                break
+            for prog in wrappers:
+                # Note: Some use OpenMPI with Intel compilers on Linux
+                result = self._try_openmpi_wrapper(prog, cid)
+                if result is not None:
+                    self.is_found = True
+                    self.version = result[0]
+                    self.compile_args = self._filter_compile_args(result[1])
+                    self.link_args = self._filter_link_args(result[2], cid)
+                    break
+                result = self._try_other_wrapper(prog, cid)
+                if result is not None:
+                    self.is_found = True
+                    self.version = result[0]
+                    self.compile_args = self._filter_compile_args(result[1])
+                    self.link_args = self._filter_link_args(result[2], cid)
+                    break
 
-        if not self.is_found and mesonlib.is_windows():
-            # only Intel Fortran compiler is compatible with Microsoft MPI at this time.
-            if language == 'fortran' and cid != 'intel-cl':
-                return
-            result = self._try_msmpi()
-            if result is not None:
-                self.is_found = True
-                self.version, self.compile_args, self.link_args = result
+            if not self.is_found and mesonlib.is_windows():
+                # only Intel Fortran compiler is compatible with Microsoft MPI at this time.
+                if language == 'fortran' and cid != 'intel-cl':
+                    return
+                result = self._try_msmpi()
+                if result is not None:
+                    self.is_found = True
+                    self.version, self.compile_args, self.link_args = result
+            return
 
     def _filter_compile_args(self, args: typing.Sequence[str]) -> typing.List[str]:
         """
@@ -272,3 +273,7 @@ class MPIDependency(ExternalDependency):
             return (None,
                     ['-I' + incdir, '-I' + os.path.join(incdir, post)],
                     [os.path.join(libdir, 'msmpi.lib')])
+
+    @staticmethod
+    def get_methods():
+        return [DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]

mesonbuild/dependencies/scalapack.py

@@ -21,14 +21,14 @@ from .base import CMakeDependency, DependencyMethods, ExternalDependency, PkgCon
 
 class ScalapackDependency(ExternalDependency):
     def __init__(self, environment, kwargs: dict):
-        methods = mesonlib.listify(kwargs.get('method', 'auto'))
         super().__init__('scalapack', environment, None, kwargs)
         kwargs['required'] = False
         kwargs['silent'] = True
         self.is_found = False
         self.static = kwargs.get('static', False)
+        methods = mesonlib.listify(self.methods)
 
-        if set(methods).intersection(['auto', 'pkg-config']):
+        if set([DependencyMethods.AUTO, DependencyMethods.PKGCONFIG]).intersection(methods):
             pkgconfig_files = []
             mklroot = None
             is_gcc = self.clib_compiler.get_id() == 'gcc'
@@ -106,8 +106,8 @@ class ScalapackDependency(ExternalDependency):
                     self.pcdep = pkgdep
                     return
 
-        if set(methods).intersection(['auto', 'cmake']):
-            cmakedep = CMakeDependency('Scalapack', environment, kwargs)
+        if set([DependencyMethods.AUTO, DependencyMethods.CMAKE]).intersection(methods):
+            cmakedep = CMakeDependency('Scalapack', environment, kwargs, language=self.language)
             if cmakedep.found():
                 self.compile_args = cmakedep.get_compile_args()
                 self.link_args = cmakedep.get_link_args()

test cases/fortran/13 coarray/meson.build

@@ -15,7 +15,7 @@ coarray = dependency('coarray')
 # this has to invoke a run of "sync all" to verify the MPI stack is functioning,
 # particularly for dynamic linking
 if fc.run('sync all; end', dependencies: coarray, name: 'Coarray link & run').returncode() != 0
-  error('The coarray stack (including MPI) did not link correctly so that a simple test could run.')
+  error('MESON_SKIP_TEST: coarray stack (including MPI) did not link correctly so that a simple test could run.')
 endif
 
 exe = executable('hello', 'main.f90',

test cases/frameworks/25 hdf5/meson.build

@@ -11,14 +11,14 @@ exec = executable('exec', 'main.c', dependencies : h5c)
 test('HDF5 C', exec, timeout: 30)
 
 # --- C++ tests
-if add_languages('cpp')
+if add_languages('cpp', required: false)
   h5cpp = dependency('hdf5', language : 'cpp', required : false, disabler: true)
   execpp = executable('execpp', 'main.cpp', dependencies : h5cpp)
   test('HDF5 C++', execpp, timeout: 30)
 endif
 
 # --- Fortran tests
-if add_languages('fortran')
+if add_languages('fortran', required: false)
   h5f = dependency('hdf5', language : 'fortran', required : false, disabler: true)
   exef = executable('exef', 'main.f90', dependencies : h5f)
   test('HDF5 Fortran', exef, timeout: 30)

test cases/frameworks/26 netcdf/meson.build

@@ -1,31 +1,32 @@
-project('netcdf_test', 'c', 'cpp')
-
-
-# --- C tests
-nc_c = dependency('netcdf', language : 'c', required : false)
+project('netcdf_test', 'c')
+
+cc = meson.get_compiler('c')
+c_code = '''#include <netcdf.h>
+int main(void) { return 0; }'''
+# --- C
+nc_c = dependency('netcdf', language : 'c', required : false, disabler: true)
+if not cc.links(c_code, dependencies: nc_c, name: 'NetCDF C')
+  nc_c = disabler()
+endif
 if not nc_c.found()
   error('MESON_SKIP_TEST: NetCDF C library not found, skipping NetCDF framework tests.')
 endif
 exec = executable('exec', 'main.c', dependencies : nc_c)
+test('NetCDF C', exec, timeout: 15)
 
-test('NetCDF C', exec)
-
-# --- C++ tests
-nc_cpp = dependency('netcdf', language : 'cpp', required : false)
-if nc_cpp.found()
+# --- C++
+if add_languages('cpp', required: false)
+  nc_cpp = dependency('netcdf', language : 'cpp', required : false, disabler: true)
   execpp = executable('execpp', 'main.cpp', dependencies : nc_cpp)
-  test('NetCDF C++', execpp)
+  test('NetCDF C++', execpp, timeout: 15)
 endif
 
-# --- Fortran tests
+# --- Fortran
 if build_machine.system() != 'windows' and build_machine.system() != 'darwin'
-  add_languages('fortran')
-
-  nc_f = dependency('netcdf', language : 'fortran', required : false)
-  if nc_f.found()
+  if add_languages('fortran', required: false)
+    nc_f = dependency('netcdf', language : 'fortran', required : false, disabler: true)
     exef = executable('exef', 'main.f90', dependencies : nc_f)
-
-    test('NetCDF Fortran', exef)
+    test('NetCDF Fortran', exef, timeout: 15)
   endif
 endif
 

test cases/frameworks/30 scalapack/meson.build

@@ -5,7 +5,6 @@ if not mpi_c.found()
   error('MESON_SKIP_TEST: MPI library not available')
 endif
 
-
 scalapack = dependency('scalapack', required: false)
 if not scalapack.found()
   error('MESON_SKIP_TEST: scalapack library not available')
@@ -15,7 +14,7 @@ c_exe = executable('scalapack_c', 'main.c',
   dependencies: [scalapack, mpi_c])
 test('scalapack_c', c_exe, timeout: 30)
 
-if add_languages('fortran')
+if add_languages('fortran', required: false)
   mpi_f = dependency('mpi', language: 'fortran', required: false)
   if not mpi_f.found()
     error('MESON_SKIP_TEST: MPI Fortran library not available')