Merge pull request #8479 from jtattermusch/run_python_aarch64_tests

Run python aarch64 tests on the CI

kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh

@@ -5,11 +5,18 @@ set -e
 # go to the repo root
 cd $(dirname $0)/../../../..
 
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
 # running dockcross image without any arguments generates a wrapper
 # scripts that can be used to run commands under the dockcross image
 # easily.
 # See https://github.com/dockcross/dockcross#usage for details
-docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
+docker run $DOCKER_TTY_ARGS --rm dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
 chmod +x dockcross-manylinux2014-aarch64.sh
 
 # the wrapper script has CRLF line endings and bash doesn't like that

kokoro/linux/aarch64/python_crosscompile_aarch64.sh

@@ -1,21 +1,28 @@
 #!/bin/bash
 #
-# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
+# Builds protobuf python including the C++ extension with aarch64 crosscompiler.
-# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+# The outputs of this script are laid out so that we can later test them under an aarch64 emulator.
+# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image.
 
 set -ex
 
 PYTHON="/opt/python/cp38-cp38/bin/python"
 
 ./autogen.sh
-CXXFLAGS="-fPIC -g -O2" ./configure
+CXXFLAGS="-fPIC -g -O2" ./configure --host=aarch64
 make -j8
 
+# create a simple shell wrapper that runs crosscompiled protoc under qemu
+echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh
+chmod ugo+x protoc_qemu_wrapper.sh
+
+# PROTOC variable is by build_py step that runs under ./python directory
+export PROTOC=../protoc_qemu_wrapper.sh
+
 pushd python
 
-# TODO: currently this step relies on qemu being registered with binfmt_misc so that
+# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files.
-# aarch64 binaries are automatically run with an emulator. This works well once
-# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine.
 ${PYTHON} setup.py build_py
 
 # when crosscompiling for aarch64, --plat-name needs to be set explicitly

kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64

kokoro/linux/aarch64/test_python_aarch64.sh

@@ -5,6 +5,13 @@ set -e
 # go to the repo root
 cd $(dirname $0)/../../..
 
+if [[ -t 0 ]]; then
+  DOCKER_TTY_ARGS="-it"
+else
+  # The input device on kokoro is not a TTY, so -it does not work.
+  DOCKER_TTY_ARGS=
+fi
+
 # crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
 kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
 
@@ -16,4 +23,4 @@ kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh ko
 #   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
 #   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
 #   we just run map the user's home to a throwaway temporary directory
-docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

kokoro/linux/python_aarch64/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_python_aarch64.sh

kokoro/linux/python_aarch64/continuous.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}

kokoro/linux/python_aarch64/presubmit.cfg

@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "**/sponge_log.xml"
+  }
+}