Merge pull request #9005 from jon-turney/xskip-ci-2

Add expected skip annotations for non-linux CI runs to framework tests

.github/workflows/cygwin.yml

@@ -24,6 +24,8 @@ jobs:
         include:
           - NAME: gccx64ninja
             ARCH: x86_64
+    env:
+      MESON_CI_JOBNAME: cygwin-${{ matrix.NAME }}
 
     steps:
       # cache should be saved on failure, but the action doesn't support that

.github/workflows/macos.yml

@@ -43,8 +43,18 @@ jobs:
   project-tests-appleclang:
     runs-on: macos-latest
     strategy:
+      fail-fast: false
       matrix:
-        unity: ["on", "off"]
+        include:
+        - NAME: macos-clang
+          unity: off
+        - NAME: macos-clang-unity
+          unity: on
+
+    name: ${{ matrix.NAME }}
+    env:
+      MESON_CI_JOBNAME: ${{ matrix.NAME }}
+
     steps:
     - uses: actions/checkout@v2
     # use python3 from homebrew because it is a valid framework, unlike the actions one:

.github/workflows/msys2.yml

@@ -40,6 +40,8 @@ jobs:
             MSYS2_CURSES:
             COMPILER: clang
             TOOLCHAIN: clang
+    env:
+      MESON_CI_JOBNAME: msys2-${{ matrix.NAME }}
 
     defaults:
       run:

.github/workflows/nonative.yml

@@ -22,6 +22,9 @@ jobs:
   cross-only-armhf:
     runs-on: ubuntu-latest
     container: mesonbuild/eoan:latest
+    env:
+      MESON_CI_JOBNAME: ubuntu-${{ github.job }}
+
     steps:
     - run: |
         apt-get -y purge clang gcc gdc

.github/workflows/unusedargs_missingreturn.yml

@@ -47,6 +47,8 @@ jobs:
         sudo apt install -yq --no-install-recommends g++ gfortran ninja-build gobjc gobjc++
         python -m pip install coverage codecov
     - run: ./tools/run_with_cov.py run_project_tests.py --only cmake common fortran platform-linux "objective c" "objective c++"
+      env:
+        MESON_CI_JOBNAME: linux-ubuntu-gcc-werror
     - name: Upload coverage report
       run: ./ci/upload_cov.sh "UnusedMissingReturn"
 
@@ -65,6 +67,7 @@ jobs:
         CC: gcc
         CXX: g++
         FC: gfortran
+        MESON_CI_JOBNAME: msys2-gcc-werror
 
     - name: Upload coverage report
       run: ./ci/upload_cov.sh "UnusedMissingReturn Windows"

ci/azure-steps.yml

@@ -3,6 +3,8 @@ steps:
   inputs:
     targetType: 'filePath'
     filePath: .\ci\run.ps1
+  env:
+    MESON_CI_JOBNAME: azure-$(System.JobName)
 
 - task: PublishTestResults@2
   inputs:

run_project_tests.py

@@ -962,12 +962,6 @@ def skip_dont_care(t: TestDef) -> bool:
     if not t.category.endswith('frameworks'):
         return True
 
-    # For the moment, all skips in jobs which don't set MESON_CI_JOBNAME are
-    # treated as expected.  In the future, we should make it mandatory to set
-    # MESON_CI_JOBNAME for all CI jobs.
-    if ci_jobname is None:
-        return True
-
     return False
 
 def skip_csharp(backend: Backend) -> bool:
@@ -1490,6 +1484,9 @@ def clear_transitive_files() -> None:
             mesonlib.windows_proof_rm(str(d))
 
 if __name__ == '__main__':
+    if under_ci and not ci_jobname:
+        raise SystemExit('Running under CI but MESON_CI_JOBNAME is not set')
+
     setup_vsenv()
 
     try:

test cases/frameworks/1 boost/test.json

@@ -17,5 +17,6 @@
       { "static": "false", "b_vscrt": "mt"  },
       { "static": "false", "b_vscrt": "mtd" }
     ]
-  }
+  },
+  "skip_on_jobname": ["azure", "msys2"]
 }

test cases/frameworks/10 gtk-doc/test.json

@@ -57,5 +57,6 @@
     {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/style.css"},
     {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up.png"},
     {"type": "file", "file": "usr/share/gtk-doc/html/foobar3/up-insensitive.png"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "macos", "msys2"]
 }

test cases/frameworks/11 gir subproject/test.json

@@ -8,5 +8,6 @@
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirlib.dll.a"},
     {"type": "expr", "file": "usr/lib/?libgirlib.so"},
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/12 multiple gir/test.json

@@ -8,5 +8,6 @@
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirsubproject.dll.a"},
     {"type": "file", "file": "usr/share/gir-1.0/Meson-1.0.gir"},
     {"type": "file", "file": "usr/share/gir-1.0/MesonSub-1.0.gir"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "macos", "msys2"]
 }

test cases/frameworks/13 yelp/test.json

@@ -18,5 +18,6 @@
     {"type": "file", "file": "usr/share/help/es/meson-linguas/index.page"},
     {"type": "file", "file": "usr/share/help/de/meson-linguas/index.page"},
     {"type": "file", "file": "usr/share/help/de/meson-linguas/media/test.txt"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/14 doxygen/test.json

@@ -1,5 +1,6 @@
 {
   "installed": [
     {"type": "dir", "file": "usr/share/doc/spede/html"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/15 llvm/test.json

@@ -2,8 +2,8 @@
   "matrix": {
     "options": {
       "method": [
-        { "val": "config-tool" },
-        { "val": "cmake" }
+        { "val": "config-tool", "skip_on_jobname": ["msys2-gcc"]},
+        { "val": "cmake", "skip_on_jobname": ["msys2"] }
       ],
       "link-static": [
         { "val": true, "skip_on_jobname": ["opensuse"] },
@@ -13,5 +13,6 @@
     "exclude": [
       { "method": "cmake", "link-static": false }
     ]
-  }
+  },
+  "skip_on_jobname": ["azure", "cygwin"]
 }

test cases/frameworks/16 sdl2/test.json

@@ -6,8 +6,9 @@
         { "val": "pkg-config" },
         { "val": "config-tool" },
         { "val": "sdlconfig" },
-        { "val": "extraframework", "skip_on_os": ["!macos"] }
+        { "val": "extraframework", "skip_on_os": ["!darwin"], "skip_on_jobname": ["macos"] }
       ]
     }
-  }
+  },
+  "skip_on_jobname": ["azure", "cygwin", "msys2"]
 }

test cases/frameworks/17 mpi/test.json

@@ -13,5 +13,5 @@
       ]
     }
   },
-  "skip_on_jobname": ["opensuse"]
+  "skip_on_jobname": ["azure", "cygwin", "msys2", "opensuse"]
 }

test cases/frameworks/18 vulkan/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/19 pcap/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "msys2"]
+}

test cases/frameworks/2 gtest/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/20 cups/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "msys2"]
+}

test cases/frameworks/21 libwmf/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/22 gir link order/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "macos", "msys2"]
+}

test cases/frameworks/23 hotdoc/test.json

@@ -4,5 +4,6 @@
   ],
   "tools": {
     "hotdoc": ">=0.1.0"
-  }
+  },
+  "skip_on_jobname": ["msys2"]
 }

test cases/frameworks/24 libgcrypt/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure"]
+}

test cases/frameworks/25 hdf5/test.json

@@ -2,10 +2,10 @@
   "matrix": {
     "options": {
       "method": [
-        { "val": "pkg-config" },
+        { "val": "pkg-config", "skip_on_jobname": ["macos"] },
         { "val": "config-tool" }
       ]
     }
   },
-  "skip_on_jobname": ["fedora", "opensuse"]
+  "skip_on_jobname": ["azure", "cygwin", "fedora", "msys2", "opensuse"]
 }

test cases/frameworks/26 netcdf/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["bionic", "fedora", "opensuse", "ubuntu"]
+  "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "macos", "msys2", "opensuse", "ubuntu"]
 }

test cases/frameworks/27 gpgme/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/28 gir link order 2/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "macos", "msys2"]
+}

test cases/frameworks/29 blocks/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["gcc"]
+  "skip_on_jobname": ["azure", "gcc", "msys2"]
 }

test cases/frameworks/3 gmock/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/30 scalapack/test.json

@@ -1,3 +1,3 @@
 {
-  "skip_on_jobname": ["bionic", "fedora", "opensuse"]
+  "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"]
 }

test cases/frameworks/31 curses/test.json

@@ -3,9 +3,10 @@
     "options": {
       "method": [
         { "val": "pkg-config" },
-        { "val": "config-tool" },
-        { "val": "system" }
+        { "val": "config-tool", "skip_on_jobname": ["msys2"] },
+        { "val": "system", "skip_on_os": ["windows"] }
       ]
     }
-  }
+  },
+  "skip_on_jobname": ["azure", "cygwin"]
 }

test cases/frameworks/34 gir static lib/test.json

@@ -5,5 +5,5 @@
     {"type": "file", "platform": "cygwin", "file": "usr/lib/libgirlib.dll.a"},
     {"type": "file", "file": "usr/share/gir-1.0/Meson-1.0.gir"}
   ],
-  "skip_on_jobname": ["bionic"]
+  "skip_on_jobname": ["azure", "bionic", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/4 qt/test.json

@@ -7,5 +7,6 @@
         { "val": "pkg-config" }
       ]
     }
-  }
+  },
+  "skip_on_jobname": ["cygwin", "msys2", "azure"]
 }

test cases/frameworks/5 protocol buffers/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
+}

test cases/frameworks/6 gettext/test.json

@@ -10,5 +10,6 @@
     {"type": "file", "file": "usr/share/applications/test2.desktop"},
     {"type": "file", "file": "usr/share/applications/test3.desktop"},
     {"type": "file", "file": "usr/share/applications/test4.desktop"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "cygwin"]
 }

test cases/frameworks/7 gnome/test.json

@@ -28,5 +28,6 @@
     {"type": "file", "file": "usr/include/enums6.h"},
     {"type": "file", "file": "usr/include/simple-resources.h"},
     {"type": "file", "file": "usr/include/generated-gdbus.h"}
-  ]
+  ],
+  "skip_on_jobname": ["azure", "cygwin", "macos", "msys2"]
 }

test cases/frameworks/8 flex/test.json

@@ -0,0 +1,3 @@
+{
+  "skip_on_jobname": ["azure", "cygwin"]
+}