Annotate framework tests with where they are expected to skip

Remove hard-coded framework test skip logic in skippable(), instead
annotate test.json with environments in which skip is expected.

(Mainly this is done with by testing the value of MESON_CI_JOBNAME now
set for linux jobs)

.github/workflows/os_comp.yml

@@ -1,4 +1,4 @@
-name: OS Comp Tests
+name: linux
 
 on:
   push:
@@ -34,6 +34,9 @@ jobs:
           - { name: OpenSUSE,         id: opensuse }
           - { name: Ubuntu Bionic,    id: bionic   }
     container: mesonbuild/${{ matrix.cfg.id }}:latest
+    env:
+      MESON_CI_JOBNAME: linux-${{ matrix.cfg.id }}-gcc
+
     steps:
     - uses: actions/checkout@v2
     - name: Run tests
@@ -77,6 +80,9 @@ jobs:
             CC: 'gcc'
             CXX: 'g++'
 
+    env:
+      MESON_CI_JOBNAME: linux-${{ github.job }}-${{ matrix.cfg.CC }}
+
     container:
       image: mesonbuild/ubuntu-rolling
       env:

ci/ciimage/bionic/image.json

@@ -2,7 +2,6 @@
   "base_image": "ubuntu:bionic",
   "env": {
     "CI": "1",
-    "SKIP_SCIENTIFIC": "1",
     "DC": "gdc"
   }
 }

ci/ciimage/fedora/image.json

@@ -2,7 +2,6 @@
   "base_image": "fedora:latest",
   "env": {
     "CI":                "1",
-    "SKIP_SCIENTIFIC":   "1",
     "SKIP_STATIC_BOOST": "1"
   }
 }

ci/ciimage/opensuse/image.json

@@ -2,7 +2,6 @@
   "base_image": "opensuse/tumbleweed:latest",
   "env": {
     "CI":                  "1",
-    "SKIP_SCIENTIFIC":     "1",
     "SKIP_STATIC_BOOST":   "1",
     "SINGLE_DUB_COMPILER": "1"
   }

run_project_tests.py

@@ -265,7 +265,6 @@ class TestDef:
 failing_logs: T.List[str] = []
 print_debug = 'MESON_PRINT_TEST_OUTPUT' in os.environ
 under_ci = 'CI' in os.environ
-skip_scientific = under_ci and ('SKIP_SCIENTIFIC' in os.environ)
 ci_jobname = os.environ.get('MESON_CI_JOBNAME', None)
 do_debug = under_ci or print_debug
 no_meson_log_msg = 'No meson-log.txt found.'
@@ -955,8 +954,6 @@ def have_java() -> bool:
     return False
 
 def skip_dont_care(t: TestDef) -> bool:
-    test = t.path.as_posix()
-
     # Everything is optional when not running on CI
     if not under_ci:
         return True
@@ -971,59 +968,7 @@ def skip_dont_care(t: TestDef) -> bool:
     if ci_jobname is None:
         return True
 
-    # gtk-doc test may be skipped, pending upstream fixes for spaces in
-    # filenames landing in the distro used for CI
-    if test.endswith('10 gtk-doc'):
-        return True
-
-    # NetCDF is not in the CI Docker image
-    if test.endswith('netcdf'):
-        return True
-
-    # Blocks are not supported on all compilers
-    if test.endswith('29 blocks'):
-        return True
-
-    # Scientific libraries are skippable on certain systems
-    # See the discussion here: https://github.com/mesonbuild/meson/pull/6562
-    if any([x in test for x in ['17 mpi', '25 hdf5', '30 scalapack']]) and skip_scientific:
-        return True
-
-    # These create OS specific tests, and need to be skippable
-    if any([x in test for x in ['16 sdl', '17 mpi']]):
-        return True
-
-    # We test cmake, and llvm-config. Some linux spins don't provide cmake or
-    # don't provide either the static or shared llvm libraries (fedora and
-    # opensuse only have the dynamic ones, for example).
-    if test.endswith('15 llvm'):
-        return True
-
-    # This test breaks with gobject-introspection <= 1.58.1
-    if test.endswith('34 gir static lib'):
-        return True
-
-    # Boost test should only be skipped for windows CI build matrix entries
-    # which don't define BOOST_ROOT
-    if test.endswith('1 boost'):
-        if mesonlib.is_windows():
-            return 'BOOST_ROOT' not in os.environ
-        return False
-
-    # Not all OSes have all of the methods for qt (qmake and pkg-config), don't
-    # fail if that happens.
-    #
-    # On macOS we should have all of the requirements at all times.
-    if test.endswith('4 qt'):
-        return not mesonlib.is_osx()
-
-    # No frameworks test should be skipped on linux CI, as we expect all
-    # prerequisites to be installed
-    if mesonlib.is_linux():
-        return False
-
-    # Other framework tests are allowed to be skipped on other platforms
-    return True
+    return False
 
 def skip_csharp(backend: Backend) -> bool:
     if backend is not Backend.ninja:

test cases/frameworks/15 llvm/test.json

@@ -6,7 +6,7 @@
         { "val": "cmake" }
       ],
       "link-static": [
-        { "val": true },
+        { "val": true, "skip_on_jobname": ["opensuse"] },
         { "val": false }
       ]
     },

test cases/frameworks/16 sdl2/test.json

@@ -6,7 +6,7 @@
         { "val": "pkg-config" },
         { "val": "config-tool" },
         { "val": "sdlconfig" },
-        { "val": "extraframework" }
+        { "val": "extraframework", "skip_on_os": ["!macos"] }
       ]
     }
   }

test cases/frameworks/17 mpi/test.json

@@ -4,12 +4,14 @@
       "method": [
         { "val": "auto" },
         { "val": "pkg-config" },
-        { "val": "config-tool" },
+        { "val": "config-tool",
+          "skip_on_jobname": ["fedora"] },
         {
           "val": "system",
           "compilers": { "c" :"msvc", "cpp": "msvc" }
         }
       ]
     }
-  }
+  },
+  "skip_on_jobname": ["opensuse"]
 }

test cases/frameworks/25 hdf5/test.json

@@ -6,5 +6,6 @@
         { "val": "config-tool" }
       ]
     }
-  }
+  },
+  "skip_on_jobname": ["fedora", "opensuse"]
 }

test cases/frameworks/26 netcdf/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["bionic", "fedora", "opensuse", "ubuntu"]
+}

test cases/frameworks/29 blocks/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["gcc"]
+}

test cases/frameworks/30 scalapack/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["bionic", "fedora", "opensuse"]
+}

test cases/frameworks/34 gir static lib/test.json

@@ -4,5 +4,6 @@
     {"type": "expr", "file": "usr/lib/?libgirlib.so"},
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirlib.dll.a"},
     {"type": "file", "file": "usr/share/gir-1.0/Meson-1.0.gir"}
-  ]
+  ],
+  "skip_on_jobname": ["bionic"]
 }

test cases/frameworks/4 qt/test.json

@@ -2,8 +2,8 @@
   "matrix": {
     "options": {
       "method": [
-        { "val": "config-tool" },
-        { "val": "qmake" },
+        { "val": "config-tool", "skip_on_jobname": ["fedora", "opensuse"] },
+        { "val": "qmake", "skip_on_jobname": ["fedora", "opensuse"] },
         { "val": "pkg-config" }
       ]
     }