Merge pull request #6843 from mensinda/ciJSONSCHEMA

CI: Maintain docker images with GitHub Actions

.github/workflows/images.yml

@@ -0,0 +1,54 @@
+name: CI image builder
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'ci/ciimage/**'
+      - '.github/workflows/images.yml'
+
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'ci/ciimage/**'
+      - '.github/workflows/images.yml'
+
+  # Rebuild the images every week (Sunday)
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  build:
+    name: ${{ matrix.cfg.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cfg:
+          - { name: Arch Linux,       id: arch     }
+          - { name: Fedora,           id: fedora   }
+          - { name: OpenSUSE,         id: opensuse }
+          - { name: Ubuntu Eoan,      id: eoan     }
+    steps:
+      - uses: actions/checkout@v2
+
+      # Login to dockerhub
+      - name: Docker login
+        if:   github.event_name == 'push' || github.event_name == 'schedule'
+        uses: azure/docker-login@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Build and test
+      - name: Building the ${{ matrix.cfg.id }} image
+        run:  ./ci/ciimage/build.py -t build ${{ matrix.cfg.id }}
+      - name: Testing the ${{ matrix.cfg.id }} image
+        run:  ./ci/ciimage/build.py -t test  ${{ matrix.cfg.id }}
+
+      # Publish
+      - name: Push the ${{ matrix.cfg.id }} image
+        if:   github.event_name == 'push' || github.event_name == 'schedule'
+        run:  docker push mesonbuild/${{ matrix.cfg.id }}

.github/workflows/os_comp.yml

@@ -1,6 +1,10 @@
 name: OS Comp Tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
 
 jobs:
   xenial:
@@ -25,38 +29,19 @@ jobs:
         XENIAL: '1'
 
   arch:
-    name: Arch Linux
+    name: ${{ matrix.cfg.name }}
     runs-on: ubuntu-latest
-    container: mensinda/arch:latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cfg:
+          - { name: Arch Linux, id: arch     }
+          - { name: Fedora,     id: fedora   }
+          - { name: OpenSUSE,   id: opensuse }
+    container: mesonbuild/${{ matrix.cfg.id }}:latest
     steps:
     - uses: actions/checkout@v1
     - name: Run tests
-      run: ./run_tests.py
+      # All environment variables are stored inside the docker image in /ci/env_vars.sh
-      env:
+      # They are defined in the `env` section in each image.json
-        CI: '1'
+      run: bash -c "source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py"
-
-  fedora:
-    name: Fedora
-    runs-on: ubuntu-latest
-    container: mensinda/fedora:latest
-    steps:
-    - uses: actions/checkout@v1
-    - name: Run tests
-      run: ./run_tests.py
-      env:
-        CI: '1'
-        SKIP_SCIENTIFIC: '1'
-        SKIP_STATIC_BOOST: '1'
-
-  opensuse:
-    name: OpenSUSE
-    runs-on: ubuntu-latest
-    container: mensinda/opensuse:latest
-    steps:
-    - uses: actions/checkout@v1
-    - name: Run tests
-      run: ./run_tests.py
-      env:
-        CI: '1'
-        SKIP_SCIENTIFIC: '1'
-        SKIP_STATIC_BOOST: '1'

ci/ciimage/.gitignore

@@ -0,0 +1,2 @@
+/build_*
+/test_*

ci/ciimage/arch/Dockerfile

@@ -1,4 +0,0 @@
-FROM archlinux:latest
-
-ADD install.sh /usr/sbin/docker-arch-install
-RUN docker-arch-install

ci/ciimage/arch/image.json

@@ -0,0 +1,6 @@
+{
+  "base_image": "archlinux:latest",
+  "env": {
+    "CI": "1"
+  }
+}

ci/ciimage/arch/install.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
+set -e
+
 # Inspired by https://github.com/greyltc/docker-archlinux-aur/blob/master/add-aur.sh
 
 pkgs=(
-  python python-setuptools python-wheel python-pip python-pytest-xdist python-gobject
+  python python-setuptools python-wheel python-pip python-pytest-xdist python-gobject python-jsonschema
   ninja make git sudo fakeroot autoconf automake patch
   libelf gcc gcc-fortran gcc-objc vala rust bison flex cython go dlang-dmd
   mono boost qt5-base gtkmm3 gtest gmock protobuf wxgtk gobject-introspection
@@ -13,7 +15,8 @@ pkgs=(
   # cuda
 )
 
-aur_pkgs=(hotdoc scalapack)
+aur_pkgs=(scalapack)
+pip_pkgs=(hotdoc)
 cleanup_pkgs=(go)
 
 AUR_USER=docker
@@ -26,6 +29,7 @@ sed -i "s,PKGEXT='.pkg.tar.xz',PKGEXT='.pkg.tar',g"  /etc/makepkg.conf
 
 # Install packages
 pacman -Syu $PACMAN_OPTS "${pkgs[@]}"
+python -m pip install "${pip_pkgs[@]}"
 
 # Setup the user
 useradd -m $AUR_USER

ci/ciimage/build.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+import json
+import argparse
+import stat
+import textwrap
+import shutil
+import subprocess
+from tempfile import TemporaryDirectory
+from pathlib import Path
+import typing as T
+
+image_namespace = 'mesonbuild'
+
+image_def_file = 'image.json'
+install_script = 'install.sh'
+
+class ImageDef:
+    def __init__(self, image_dir: Path) -> None:
+        path = image_dir / image_def_file
+        data = json.loads(path.read_text())
+
+        assert isinstance(data, dict)
+        assert all([x in data for x in ['base_image', 'env']])
+        assert isinstance(data['base_image'], str)
+        assert isinstance(data['env'],  dict)
+
+        self.base_image: str = data['base_image']
+        self.env: T.Dict[str, str] = data['env']
+
+class BuilderBase():
+    def __init__(self, data_dir: Path, temp_dir: Path) -> None:
+        self.data_dir = data_dir
+        self.temp_dir = temp_dir
+
+        self.common_sh = self.data_dir.parent / 'common.sh'
+        self.common_sh = self.common_sh.resolve(strict=True)
+        self.validate_data_dir()
+
+        self.image_def = ImageDef(self.data_dir)
+
+        self.docker = shutil.which('docker')
+        self.git = shutil.which('git')
+        if self.docker is None:
+            raise RuntimeError('Unable to find docker')
+        if self.git is None:
+            raise RuntimeError('Unable to find git')
+
+    def validate_data_dir(self) -> None:
+        files = [
+            self.data_dir / image_def_file,
+            self.data_dir / install_script,
+        ]
+        if not self.data_dir.exists():
+            raise RuntimeError(f'{self.data_dir.as_posix()} does not exist')
+        for i in files:
+            if not i.exists():
+                raise RuntimeError(f'{i.as_posix()} does not exist')
+            if not i.is_file():
+                raise RuntimeError(f'{i.as_posix()} is not a regular file')
+
+class Builder(BuilderBase):
+    def gen_bashrc(self) -> None:
+        out_file = self.temp_dir / 'env_vars.sh'
+        out_data = ''
+
+        for key, val in self.image_def.env.items():
+            out_data += f'export {key}="{val}"\n'
+
+        out_file.write_text(out_data)
+
+        # make it executable
+        mode = out_file.stat().st_mode
+        out_file.chmod(mode | stat.S_IEXEC)
+
+    def gen_dockerfile(self) -> None:
+        out_file = self.temp_dir / 'Dockerfile'
+        out_data = textwrap.dedent(f'''\
+            FROM {self.image_def.base_image}
+
+            ADD install.sh  /ci/install.sh
+            ADD common.sh   /ci/common.sh
+            ADD env_vars.sh /ci/env_vars.sh
+            RUN /ci/install.sh
+        ''')
+
+        out_file.write_text(out_data)
+
+    def do_build(self) -> None:
+        # copy files
+        for i in self.data_dir.iterdir():
+            shutil.copy(str(i), str(self.temp_dir))
+        shutil.copy(str(self.common_sh), str(self.temp_dir))
+
+        self.gen_bashrc()
+        self.gen_dockerfile()
+
+        cmd_git = [self.git, 'rev-parse', '--short', 'HEAD']
+        res = subprocess.run(cmd_git, cwd=self.data_dir, stdout=subprocess.PIPE)
+        if res.returncode != 0:
+            raise RuntimeError('Failed to get the current commit hash')
+        commit_hash = res.stdout.decode().strip()
+
+        cmd = [
+            self.docker, 'build',
+            '-t', f'{image_namespace}/{self.data_dir.name}:latest',
+            '-t', f'{image_namespace}/{self.data_dir.name}:{commit_hash}',
+            '--pull',
+            self.temp_dir.as_posix(),
+        ]
+        if subprocess.run(cmd).returncode != 0:
+            raise RuntimeError('Failde to build the docker image')
+
+class ImageTester(BuilderBase):
+    def __init__(self, data_dir: Path, temp_dir: Path, ci_root: Path) -> None:
+        super().__init__(data_dir, temp_dir)
+        self.meson_root = ci_root.parent.parent.resolve()
+
+    def gen_dockerfile(self) -> None:
+        out_file = self.temp_dir / 'Dockerfile'
+        out_data = textwrap.dedent(f'''\
+            FROM {image_namespace}/{self.data_dir.name}
+
+            ADD meson /meson
+        ''')
+
+        out_file.write_text(out_data)
+
+    def copy_meson(self) -> None:
+        shutil.copytree(
+            self.meson_root,
+            self.temp_dir / 'meson',
+            ignore=shutil.ignore_patterns(
+                '.git',
+                '*_cache',
+                'work area',
+                self.temp_dir.name,
+            )
+        )
+
+    def do_test(self):
+        self.copy_meson()
+        self.gen_dockerfile()
+
+        try:
+            build_cmd = [
+                self.docker, 'build',
+                '-t', 'meson_test_image',
+                self.temp_dir.as_posix(),
+            ]
+            if subprocess.run(build_cmd).returncode != 0:
+                raise RuntimeError('Failde to build the test docker image')
+
+            test_cmd = [
+                self.docker, 'run', '--rm', '-t', 'meson_test_image',
+                '/usr/bin/bash', '-c', 'source /ci/env_vars.sh; cd meson; ./run_tests.py'
+            ]
+            if subprocess.run(test_cmd).returncode != 0:
+                raise RuntimeError('Running tests failed')
+        finally:
+            cleanup_cmd = [self.docker, 'rmi', '-f', 'meson_test_image']
+            subprocess.run(cleanup_cmd).returncode
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description='Meson CI image builder')
+    parser.add_argument('what', type=str, help='Which image to build / test')
+    parser.add_argument('-t', '--type', choices=['build', 'test'], help='What to do', required=True)
+
+    args = parser.parse_args()
+
+    ci_root = Path(__file__).parent
+    ci_data = ci_root / args.what
+
+    with TemporaryDirectory(prefix=f'{args.type}_{args.what}_', dir=ci_root) as td:
+        ci_build = Path(td)
+        print(f'Build dir: {ci_build}')
+
+        if args.type == 'build':
+            builder = Builder(ci_data, ci_build)
+            builder.do_build()
+        elif args.type == 'test':
+            tester = ImageTester(ci_data, ci_build, ci_root)
+            tester.do_test()
+
+if __name__ == '__main__':
+    main()

ci/ciimage/common.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+###
+### Common functions for CI builder files.
+### All functions can be accessed in install.sh via:
+###
+### $ source /ci/common.sh
+###
+
+set -e
+
+dub_fetch() {
+  set +e
+  for (( i=1; i<=24; ++i )); do
+    dub fetch "$@"
+    (( $? == 0 )) && break
+
+    echo "Dub Fetch failed. Retrying in $((i*5))s"
+    sleep $((i*5))
+  done
+  set -e
+}

ci/ciimage/eoan/Dockerfile

@@ -1,36 +0,0 @@
-FROM ubuntu:eoan
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV LANG='C.UTF-8'
-ENV DC=gdc
-
-RUN sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" \
-&& apt-get -y update && apt-get -y upgrade \
-&& apt-get -y install eatmydata \
-&& eatmydata apt-get -y build-dep meson \
-&& eatmydata apt-get -y install python3-pytest-xdist \
-&& eatmydata apt-get -y install python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev \
-&& eatmydata python3 -m pip install hotdoc codecov \
-&& eatmydata apt-get -y install wget unzip \
-&& eatmydata apt-get -y install qt5-default clang \
-&& eatmydata apt-get -y install pkg-config-arm-linux-gnueabihf \
-&& eatmydata apt-get -y install qt4-linguist-tools \
-&& eatmydata apt-get -y install python-dev \
-&& eatmydata apt-get -y install libomp-dev \
-&& eatmydata apt-get -y install dub ldc \
-&& eatmydata apt-get -y install mingw-w64 mingw-w64-tools nim \
-&& eatmydata apt-get -y install --no-install-recommends wine-stable \
-&& eatmydata apt-get -y install libclang-dev \
-&& eatmydata apt-get -y install libgcrypt20-dev \
-&& eatmydata apt-get -y install libgpgme-dev \
-&& eatmydata apt-get -y install libhdf5-dev \
-&& eatmydata apt-get -y install libboost-python-dev libboost-regex-dev \
-&& eatmydata apt-get -y install libblocksruntime-dev \
-&& eatmydata apt-get -y install libperl-dev \
-&& eatmydata apt-get -y install liblapack-dev libscalapack-mpi-dev \
-&& eatmydata dub fetch urld && dub build urld --compiler=gdc \
-&& eatmydata dub fetch dubtestproject \
-&& eatmydata dub build dubtestproject:test1 --compiler=ldc2 \
-&& eatmydata dub build dubtestproject:test2 --compiler=ldc2
-# OpenSSH client is needed to run openmpi binaries.
-

ci/ciimage/eoan/image.json

@@ -0,0 +1,7 @@
+{
+  "base_image": "ubuntu:eoan",
+  "env": {
+    "CI": "1",
+    "DC": "gdc"
+  }
+}

ci/ciimage/eoan/install.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e
+
+source /ci/common.sh
+
+export DEBIAN_FRONTEND=noninteractive
+export LANG='C.UTF-8'
+export DC=gdc
+
+pkgs=(
+  python3-pytest-xdist
+  python3-pip libxml2-dev libxslt1-dev libyaml-dev libjson-glib-dev
+  wget unzip
+  qt5-default clang
+  pkg-config-arm-linux-gnueabihf
+  qt4-linguist-tools
+  python-dev
+  libomp-dev
+  dub ldc
+  mingw-w64 mingw-w64-tools nim
+  libclang-dev
+  libgcrypt20-dev
+  libgpgme-dev
+  libhdf5-dev
+  libboost-python-dev libboost-regex-dev
+  libblocksruntime-dev
+  libperl-dev
+  liblapack-dev libscalapack-mpi-dev
+)
+
+sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list"
+apt-get -y update
+apt-get -y upgrade
+apt-get -y install eatmydata
+
+# Base stuff
+eatmydata apt-get -y build-dep meson
+
+# packages
+eatmydata apt-get -y install "${pkgs[@]}"
+eatmydata apt-get -y install --no-install-recommends wine-stable  # Wine is special
+
+eatmydata python3 -m pip install hotdoc codecov jsonschema
+
+# dub stuff
+dub_fetch urld
+dub build urld --compiler=gdc
+dub_fetch dubtestproject
+dub build dubtestproject:test1 --compiler=ldc2
+dub build dubtestproject:test2 --compiler=ldc2
+
+# cleanup
+apt-get -y clean
+apt-get -y autoclean

ci/ciimage/eoan/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+testFN() {
+  set +e
+  false
+}
+
+testFN
+false
+exit 0

ci/ciimage/fedora/Dockerfile

@@ -1,4 +0,0 @@
-FROM fedora:latest
-
-ADD install.sh /usr/sbin/docker-fedora-install
-RUN docker-fedora-install

ci/ciimage/fedora/image.json

@@ -0,0 +1,8 @@
+{
+  "base_image": "fedora:latest",
+  "env": {
+    "CI":                "1",
+    "SKIP_SCIENTIFIC":   "1",
+    "SKIP_STATIC_BOOST": "1"
+  }
+}

ci/ciimage/fedora/install.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 
+set -e
+
 pkgs=(
   python python-setuptools python-wheel python-pip python-pytest-xdist pygobject3 python3-devel python2-devel
-  ninja-build make git autoconf automake patch python3-Cython python2-Cython
+  ninja-build make git autoconf automake patch python3-Cython python2-Cython python3-jsonschema
   elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-objc++ vala rust bison flex ldc libasan libasan-static
   mono-core boost-devel gtkmm30 gtest-devel gmock-devel protobuf-devel wxGTK3-devel gobject-introspection
   boost-python3-devel boost-python2-devel

ci/ciimage/opensuse/Dockerfile

@@ -1,4 +0,0 @@
-FROM opensuse/tumbleweed:latest
-
-ADD install.sh /usr/sbin/docker-opensuse-install
-RUN docker-opensuse-install

ci/ciimage/opensuse/image.json

@@ -0,0 +1,9 @@
+{
+  "base_image": "opensuse/tumbleweed:latest",
+  "env": {
+    "CI":                  "1",
+    "SKIP_SCIENTIFIC":     "1",
+    "SKIP_STATIC_BOOST":   "1",
+    "SINGLE_DUB_COMPILER": "1"
+  }
+}

ci/ciimage/opensuse/install.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
+set -e
+
+source /ci/common.sh
+
 pkgs=(
-  python3-setuptools python3-wheel python3-pip python3-pytest-xdist python3 python2
+  python3-setuptools python3-wheel python3-pip python3-pytest-xdist python3
-  ninja make git autoconf automake patch python3-Cython python2-Cython
+  ninja make git autoconf automake patch python3-Cython python3-jsonschema
   elfutils gcc gcc-c++ gcc-fortran gcc-objc gcc-obj-c++ vala rust bison flex curl
   mono-core gtkmm3-devel gtest gmock protobuf-devel wxGTK3-3_2-devel gobject-introspection-devel
-  itstool gtk3-devel java-13-openjdk-devel gtk-doc llvm-devel clang-devel libSDL2-devel graphviz-devel zlib-devel zlib-devel-static
+  itstool gtk3-devel java-15-openjdk-devel gtk-doc llvm-devel clang-devel libSDL2-devel graphviz-devel zlib-devel zlib-devel-static
   #hdf5-devel netcdf-devel libscalapack2-openmpi3-devel libscalapack2-gnu-openmpi3-hpc-devel openmpi3-devel
   doxygen vulkan-devel vulkan-validationlayers openssh mercurial gtk-sharp3-complete gtk-sharp2-complete libpcap-devel libgpgme-devel
   libqt5-qtbase-devel libqt5-qttools-devel libqt5-linguist libqt5-qtbase-private-headers-devel
@@ -13,32 +17,32 @@ pkgs=(
   libxml2-devel libxslt-devel libyaml-devel glib2-devel json-glib-devel
   boost-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_system-devel
   libboost_test-devel libboost_log-devel libboost_regex-devel
-  libboost_python-devel libboost_python-py2_7-1_71_0-devel libboost_python-py3-1_71_0-devel libboost_regex-devel
+  libboost_python-devel libboost_python-py3-1_71_0-devel libboost_regex-devel
 )
 
 # Sys update
-zypper patch --with-update --with-optional
+zypper --non-interactive patch --with-update --with-optional
-zypper update
+zypper --non-interactive update
 
 # Install deps
 zypper install -y "${pkgs[@]}"
 python3 -m pip install hotdoc gobject PyGObject
 
-echo 'export PKG_CONFIG_PATH="/usr/lib64/mpi/gcc/openmpi3/lib64/pkgconfig:$PKG_CONFIG_PATH"' >> ~/.bashrc
+echo 'export PKG_CONFIG_PATH="/usr/lib64/mpi/gcc/openmpi3/lib64/pkgconfig:$PKG_CONFIG_PATH"' >> /env_vars.sh
 
 # dmd is very special on OpenSUSE (as in the packages do not work)
 # see https://bugzilla.opensuse.org/show_bug.cgi?id=1162408
 curl -fsS https://dlang.org/install.sh | bash -s dmd | tee dmd_out.txt
-cat dmd_out.txt | grep source | sed 's/^[^`]*`//g' | sed 's/`.*//g' >> ~/.bashrc
+cat dmd_out.txt | grep source | sed 's/^[^`]*`//g' | sed 's/`.*//g' >> /env_vars.sh
-chmod +x ~/.bashrc
+chmod +x /env_vars.sh
 
-source ~/.bashrc
+source /env_vars.sh
 
-dub fetch urld
+dub_fetch urld
 dub build urld --compiler=dmd
-dub fetch dubtestproject
+dub_fetch dubtestproject
 dub build dubtestproject:test1 --compiler=dmd
 dub build dubtestproject:test2 --compiler=dmd
 
 # Cleanup
-zypper clean --all
+zypper --non-interactive clean --all

mesonbuild/interpreter.py

@@ -2352,7 +2352,7 @@ class Interpreter(InterpreterBase):
 
         if isinstance(item, build.CustomTarget):
             return CustomTargetHolder(item, self)
-        elif isinstance(item, (int, str, bool)) or item is None:
+        elif isinstance(item, (int, str, bool, Disabler)) or item is None:
             return item
         elif isinstance(item, build.Executable):
             return ExecutableHolder(item, self)
@@ -2397,7 +2397,7 @@ class Interpreter(InterpreterBase):
                 self.process_new_values(v.sources[0])
             elif hasattr(v, 'held_object'):
                 pass
-            elif isinstance(v, (int, str, bool)):
+            elif isinstance(v, (int, str, bool, Disabler)):
                 pass
             else:
                 raise InterpreterException('Module returned a value of unknown type.')

test cases/failing/88 dub compiler/test.json

@@ -0,0 +1,9 @@
+{
+  "matrix": {
+    "options": {
+      "warning_level": [
+        { "val": "1", "skip_on_env": [ "SINGLE_DUB_COMPILER" ] }
+      ]
+    }
+  }
+}

test cases/frameworks/1 boost/meson.build

@@ -22,9 +22,9 @@ notfound    = dependency('boost', static: s, modules : ['this_should_not_exist_o
 assert(not notfound.found())
 
 pymod = import('python')
-python2 = pymod.find_installation('python2', required: host_machine.system() == 'linux', disabler: true)
+python2 = pymod.find_installation('python2', required: false                           , disabler: true)
 python3 = pymod.find_installation('python3', required: host_machine.system() == 'linux', disabler: true)
-python2dep = python2.dependency(required: host_machine.system() == 'linux', embed: true, disabler: true)
+python2dep = python2.dependency(required: false                           , embed: true, disabler: true)
 python3dep = python3.dependency(required: host_machine.system() == 'linux', embed: true, disabler: true)
 
 # compile python 2/3 modules only if we found a corresponding python version