HDF5: make much more robust across platforms

This addresses various real-world problems with HDF5 pkg-config, including

* hdf*.pc with package versions as part of the filename
* malformed hdf*.pc missing the commonly-used HDF5 HL module

---

Additionally, this refactors more complicated dependencies such as
HDF5 and OpenMPI. This may help us deduplicate internal dependency
code in the future.

HDF5 selftest: improve platform-agnostic test

ci: init demo github action for HDF5 framework

ci Actions: hold off on MSYS2 for now [skip ci]

hdf5: ensure C libraries always included

ci: mac hdf5--use clang+gfortran

.github/workflows/frameworks.yml

@@ -0,0 +1,98 @@
+# at first, we demo HDF5 framework. More can be added.
+name: ci_frameworks
+
+on:
+  push:
+    paths:
+      - "mesonbuild/dependencies/**"
+      - ".github/workflows/frameworks.yml"
+  pull_request:
+    paths:
+      - "mesonbuild/dependencies/**"
+      - ".github/workflows/frameworks.yml"
+
+jobs:
+
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - run: python -m pip install -e .
+    - run: sudo apt install -yq --no-install-recommends ninja-build g++ gfortran libhdf5-dev
+    - run: meson setup "test cases/frameworks/25 hdf5" build
+      env:
+        FC: gfortran
+        CXX: g++
+        CC: gcc
+    - run: ninja -C build
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: Linux_Log
+        path: build/meson-logs/meson-log.txt
+    - run: meson test -C build -v
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: Linux_Test
+        path: build/meson-logs/testlog.txt
+
+  mac:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - run: python -m pip install -e .
+    - run: brew install pkg-config ninja gcc hdf5
+    - run: meson setup "test cases/frameworks/25 hdf5" build
+    - run: ninja -C build
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: Mac_Log
+        path: build/meson-logs/meson-log.txt
+    - run: meson test -C build -v
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: Mac_Test
+        path: build/meson-logs/testlog.txt
+
+  windows:
+    # as usual, start in MSYS to install packages, then switch to MINGW64 to build.
+    if: false
+    # MSYS2 GitHub Action is still being developed--appears to have PATH bugs stopping this from working.
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+    - uses: numworks/setup-msys2@v1
+      with:
+        msystem: MSYS
+    - run: msys2do pacman -S mingw64/mingw-w64-x86_64-ninja mingw64/mingw-w64-x86_64-pkg-config mingw64/mingw-w64-x86_64-gcc mingw64/mingw-w64-x86_64-gcc-fortran mingw-w64-x86_64-hdf5 --noprogressbar --noconfirm
+    - run: set MSYSTEM=MINGW64
+    - run: python -m pip install -e .
+    - run: meson setup "test cases/frameworks/25 hdf5" build
+      env:
+        FC: gfortran
+        CXX: g++
+        CC: gcc
+    - run: ninja -C build
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: MSYS_Log
+        path: build/meson-logs/meson-log.txt
+    - run: meson test -C build -v
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: MSYS2_Test
+        path: build/meson-logs/testlog.txt

.github/workflows/lint_mypy.yml

@@ -30,4 +30,4 @@ jobs:
       with:
         python-version: '3.x'
     - run: python -m pip install mypy
-    - run: mypy --follow-imports=skip mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py mesonbuild/dependencies/mpi.py
+    - run: mypy --follow-imports=skip  mesonbuild/mtest.py mesonbuild/minit.py mesonbuild/msetup.py mesonbuild/wrap tools/ mesonbuild/modules/fs.py mesonbuild/dependencies/mpi.py mesonbuild/dependencies/hdf5.py

mesonbuild/dependencies/__init__.py

@@ -14,6 +14,7 @@
 
 from .boost import BoostDependency
 from .cuda import CudaDependency
+from .hdf5 import HDF5Dependency
 from .base import (  # noqa: F401
     Dependency, DependencyException, DependencyMethods, ExternalProgram, EmptyExternalProgram, NonExistingExternalProgram,
     ExternalDependency, NotFoundDependency, ExternalLibrary, ExtraFrameworkDependency, InternalDependency,
@@ -21,7 +22,7 @@ from .base import (  # noqa: F401
 from .dev import GMockDependency, GTestDependency, LLVMDependency, ValgrindDependency
 from .coarrays import CoarrayDependency
 from .mpi import MPIDependency
-from .misc import (BlocksDependency, HDF5Dependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency)
+from .misc import (BlocksDependency, NetCDFDependency, OpenMPDependency, Python3Dependency, ThreadDependency, PcapDependency, CupsDependency, LibWmfDependency, LibGCryptDependency, GpgmeDependency, ShadercDependency)
 from .platform import AppleFrameworks
 from .ui import GLDependency, GnuStepDependency, Qt4Dependency, Qt5Dependency, SDL2Dependency, WxDependency, VulkanDependency
 

mesonbuild/dependencies/hdf5.py

@@ -0,0 +1,120 @@
+# Copyright 2013-2019 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains the detection logic for miscellaneous external dependencies.
+
+import subprocess
+from pathlib import Path
+
+from .. import mlog
+from ..mesonlib import split_args
+from .base import DependencyException, ExternalDependency, ExternalProgram, PkgConfigDependency
+
+class HDF5Dependency(ExternalDependency):
+
+    def __init__(self, environment, kwargs):
+        language = kwargs.get('language', 'c')
+        super().__init__('hdf5', environment, language, kwargs)
+        kwargs['required'] = False
+        kwargs['silent'] = True
+        self.is_found = False
+
+        # 1. pkg-config
+        pkgconfig_files = ['hdf5', 'hdf5-serial']
+        # some distros put hdf5-1.2.3.pc with version number in .pc filename.
+        ret = subprocess.run(['pkg-config', '--list-all'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                             universal_newlines=True)
+        if ret.returncode == 0:
+            for pkg in ret.stdout.split('\n'):
+                if pkg.startswith(('hdf5')):
+                    pkgconfig_files.append(pkg.split(' ', 1)[0])
+            pkgconfig_files = list(set(pkgconfig_files))  # dedupe
+
+        if language not in ('c', 'cpp', 'fortran'):
+            raise DependencyException('Language {} is not supported with HDF5.'.format(language))
+
+        for pkg in pkgconfig_files:
+            pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
+            if not pkgdep.found():
+                continue
+
+            self.compile_args = pkgdep.get_compile_args()
+            # some broken pkgconfig don't actually list the full path to the needed includes
+            newinc = []
+            for arg in self.compile_args:
+                if arg.startswith('-I'):
+                    stem = 'static' if kwargs.get('static', False) else 'shared'
+                    if (Path(arg[2:]) / stem).is_dir():
+                        newinc.append('-I' + str(Path(arg[2:]) / stem))
+            self.compile_args += newinc
+
+            # derive needed libraries by language
+            pd_link_args = pkgdep.get_link_args()
+            link_args = []
+            for larg in pd_link_args:
+                lpath = Path(larg)
+                # some pkg-config hdf5.pc (e.g. Ubuntu) don't include the commonly-used HL HDF5 libraries,
+                # so let's add them if they exist
+                # additionally, some pkgconfig HDF5 HL files are malformed so let's be sure to find HL anyway
+                if lpath.is_file():
+                    hl = []
+                    if language == 'cpp':
+                        hl += ['_hl_cpp', '_cpp']
+                    elif language == 'fortran':
+                        hl += ['_hl_fortran', 'hl_fortran', '_fortran']
+                    hl += ['_hl']  # C HL library, always needed
+
+                    suffix = '.' + lpath.name.split('.', 1)[1]  # in case of .dll.a
+                    for h in hl:
+                        hlfn = lpath.parent / (lpath.name.split('.', 1)[0] + h + suffix)
+                        if hlfn.is_file():
+                            link_args.append(str(hlfn))
+                    # HDF5 C libs are required by other HDF5 languages
+                    link_args.append(larg)
+                else:
+                    link_args.append(larg)
+
+            self.link_args = link_args
+            self.version = pkgdep.get_version()
+            self.is_found = True
+            self.pcdep = pkgdep
+            return
+
+        # 2. compiler wrapper fallback
+        wrappers = {'c': 'h5cc', 'cpp': 'h5c++', 'fortran': 'h5fc'}
+        comp_args = []
+        link_args = []
+        # have to always do C as well as desired language
+        for lang in set([language, 'c']):
+            prog = ExternalProgram(wrappers[lang], silent=True)
+            if not prog.found():
+                return
+            cmd = prog.get_command() + ['-show']
+            p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=15)
+            if p.returncode != 0:
+                mlog.debug('Command', mlog.bold(cmd), 'failed to run:')
+                mlog.debug(mlog.bold('Standard output\n'), p.stdout)
+                mlog.debug(mlog.bold('Standard error\n'), p.stderr)
+                return
+            args = split_args(p.stdout)
+            for arg in args[1:]:
+                if arg.startswith(('-I', '-f', '-D')) or arg == '-pthread':
+                    comp_args.append(arg)
+                elif arg.startswith(('-L', '-l', '-Wl')):
+                    link_args.append(arg)
+                elif Path(arg).is_file():
+                    link_args.append(arg)
+        self.compile_args = comp_args
+        self.link_args = link_args
+        self.is_found = True

mesonbuild/dependencies/misc.py

@@ -30,53 +30,6 @@ from .base import (
 )
 
 
-class HDF5Dependency(ExternalDependency):
-
-    def __init__(self, environment, kwargs):
-        language = kwargs.get('language', 'c')
-        super().__init__('hdf5', environment, language, kwargs)
-        kwargs['required'] = False
-        kwargs['silent'] = True
-        self.is_found = False
-
-        pkgconfig_files = ['hdf5']
-
-        if language not in ('c', 'cpp', 'fortran'):
-            raise DependencyException('Language {} is not supported with HDF5.'.format(language))
-
-        for pkg in pkgconfig_files:
-            try:
-                pkgdep = PkgConfigDependency(pkg, environment, kwargs, language=self.language)
-                if pkgdep.found():
-                    self.compile_args = pkgdep.get_compile_args()
-                    # derive needed libraries by language
-                    pd_link_args = pkgdep.get_link_args()
-                    link_args = []
-                    for larg in pd_link_args:
-                        lpath = Path(larg)
-                        if lpath.is_file():
-                            if language == 'cpp':
-                                link_args.append(str(lpath.parent / (lpath.stem + '_hl_cpp' + lpath.suffix)))
-                                link_args.append(str(lpath.parent / (lpath.stem + '_cpp' + lpath.suffix)))
-                            elif language == 'fortran':
-                                link_args.append(str(lpath.parent / (lpath.stem + 'hl_fortran' + lpath.suffix)))
-                                link_args.append(str(lpath.parent / (lpath.stem + '_fortran' + lpath.suffix)))
-
-                            # HDF5 C libs are required by other HDF5 languages
-                            link_args.append(str(lpath.parent / (lpath.stem + '_hl' + lpath.suffix)))
-                            link_args.append(larg)
-                        else:
-                            link_args.append(larg)
-
-                    self.link_args = link_args
-                    self.version = pkgdep.get_version()
-                    self.is_found = True
-                    self.pcdep = pkgdep
-                    break
-            except Exception:
-                pass
-
-
 class NetCDFDependency(ExternalDependency):
 
     def __init__(self, environment, kwargs):

test cases/frameworks/25 hdf5/meson.build

@@ -1,40 +1,27 @@
-project('hdf5_test', 'c', 'cpp')
+project('hdf5_framework', 'c')
-
-if build_machine.system() == 'darwin'
-  error('MESON_SKIP_TEST: HDF5 CI image not setup for OSX.')
-endif
-
-if build_machine.system() == 'cygwin'
-  error('MESON_SKIP_TEST: HDF5 CI image not setup for Cygwin.')
-endif
 
+# NOTE: all HDF5 languages must have HDF5 C library working.
 
 # --- C tests
 h5c = dependency('hdf5', language : 'c', required : false)
 if not h5c.found()
-  error('MESON_SKIP_TEST: HDF5 C library not found, skipping HDF5 framework tests.')
+  error('MESON_SKIP_TEST: HDF5 C library not found.')
 endif
 exec = executable('exec', 'main.c', dependencies : h5c)
-
+test('HDF5 C', exec, timeout: 30)
-test('HDF5 C', exec)
 
 # --- C++ tests
-h5cpp = dependency('hdf5', language : 'cpp', required : false)
+if add_languages('cpp')
-if h5cpp.found()
+  h5cpp = dependency('hdf5', language : 'cpp', required : false, disabler: true)
   execpp = executable('execpp', 'main.cpp', dependencies : h5cpp)
-  test('HDF5 C++', execpp)
+  test('HDF5 C++', execpp, timeout: 30)
 endif
 
 # --- Fortran tests
-if build_machine.system() != 'windows'
+if add_languages('fortran')
-  add_languages('fortran')
+  h5f = dependency('hdf5', language : 'fortran', required : false, disabler: true)
-
+  exef = executable('exef', 'main.f90', dependencies : h5f)
-  h5f = dependency('hdf5', language : 'fortran', required : false)
+  test('HDF5 Fortran', exef, timeout: 30)
-  if h5f.found()
-    exef = executable('exef', 'main.f90', dependencies : h5f)
-
-    test('HDF5 Fortran', exef)
-  endif
 endif
 
 # Check we can apply a version constraint