ci: Retry if dub fetch failed

.github/workflows/os_comp.yml

@@ -42,6 +42,6 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Run tests
-      # All environment variables are stored inside the docker image in /env_vars.sh
+      # All environment variables are stored inside the docker image in /ci/env_vars.sh
       # They are defined in the `env` section in each image.json
-      run: bash -c "source /env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py"
+      run: bash -c "source /ci/env_vars.sh; cd $GITHUB_WORKSPACE; ./run_tests.py"

ci/ciimage/build.py

@@ -33,6 +33,8 @@ class BuilderBase():
         self.data_dir = data_dir
         self.temp_dir = temp_dir
 
+        self.common_sh = self.data_dir.parent / 'common.sh'
+        self.common_sh = self.common_sh.resolve(strict=True)
         self.validate_data_dir()
 
         self.image_def = ImageDef(self.data_dir)
@@ -76,9 +78,10 @@ class Builder(BuilderBase):
         out_data = textwrap.dedent(f'''\
             FROM {self.image_def.base_image}
 
-            ADD install.sh  /usr/sbin/docker-do-install
-            ADD env_vars.sh /env_vars.sh
-            RUN docker-do-install
+            ADD install.sh  /ci/install.sh
+            ADD common.sh   /ci/common.sh
+            ADD env_vars.sh /ci/env_vars.sh
+            RUN /ci/install.sh
         ''')
 
         out_file.write_text(out_data)
@@ -87,6 +90,7 @@ class Builder(BuilderBase):
         # copy files
         for i in self.data_dir.iterdir():
             shutil.copy(str(i), str(self.temp_dir))
+        shutil.copy(str(self.common_sh), str(self.temp_dir))
 
         self.gen_bashrc()
         self.gen_dockerfile()
@@ -149,7 +153,7 @@ class ImageTester(BuilderBase):
 
             test_cmd = [
                 self.docker, 'run', '--rm', '-t', 'meson_test_image',
-                '/usr/bin/bash', '-c', 'source /env_vars.sh; cd meson; ./run_tests.py'
+                '/usr/bin/bash', '-c', 'source /ci/env_vars.sh; cd meson; ./run_tests.py'
             ]
             if subprocess.run(test_cmd).returncode != 0:
                 raise RuntimeError('Running tests failed')

ci/ciimage/common.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+###
+### Common functions for CI builder files.
+### All functions can be accessed in install.sh via:
+###
+### $ source /ci/common.sh
+###
+
+set -e
+
+dub_fetch() {
+  set +e
+  for (( i=1; i<=24; ++i )); do
+    dub fetch "$@"
+    (( $? == 0 )) && break
+
+    echo "Dub Fetch failed. Retrying in $((i*5))s"
+    sleep $((i*5))
+  done
+  set -e
+}

ci/ciimage/eoan/install.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source /ci/common.sh
+
 export DEBIAN_FRONTEND=noninteractive
 export LANG='C.UTF-8'
 export DC=gdc
@@ -42,9 +44,9 @@ eatmydata apt-get -y install --no-install-recommends wine-stable  # Wine is spec
 eatmydata python3 -m pip install hotdoc codecov jsonschema
 
 # dub stuff
-dub fetch urld
+dub_fetch urld
 dub build urld --compiler=gdc
-dub fetch dubtestproject
+dub_fetch dubtestproject
 dub build dubtestproject:test1 --compiler=ldc2
 dub build dubtestproject:test2 --compiler=ldc2
 

ci/ciimage/eoan/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+testFN() {
+  set +e
+  false
+}
+
+testFN
+false
+exit 0

ci/ciimage/opensuse/install.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source /ci/common.sh
+
 pkgs=(
   python3-setuptools python3-wheel python3-pip python3-pytest-xdist python3
   ninja make git autoconf automake patch python3-Cython python3-jsonschema
@@ -36,9 +38,9 @@ chmod +x /env_vars.sh
 
 source /env_vars.sh
 
-dub fetch urld
+dub_fetch urld
 dub build urld --compiler=dmd
-dub fetch dubtestproject
+dub_fetch dubtestproject
 dub build dubtestproject:test1 --compiler=dmd
 dub build dubtestproject:test2 --compiler=dmd