From 7124f2893da2bb04822a6be8aee40e5ae8a03fe2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 29 Apr 2021 16:45:34 +0200 Subject: [PATCH 1/3] add testing scripts for protobuf javascript on aarch64 qemu emulator --- ...t_build_and_run_tests_with_qemu_aarch64.sh | 10 +++++++ .../linux/aarch64/test_javascript_aarch64.sh | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh create mode 100755 kokoro/linux/aarch64/test_javascript_aarch64.sh diff --git a/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh b/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh new file mode 100755 index 0000000000..e5d07e8bd2 --- /dev/null +++ b/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +# go to the repo root +cd $(dirname $0)/../../.. + +cd js +npm install +npm test diff --git a/kokoro/linux/aarch64/test_javascript_aarch64.sh b/kokoro/linux/aarch64/test_javascript_aarch64.sh new file mode 100755 index 0000000000..dcddd60a2e --- /dev/null +++ b/kokoro/linux/aarch64/test_javascript_aarch64.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +# go to the repo root +cd $(dirname $0)/../../.. + +if [[ -t 0 ]]; then + DOCKER_TTY_ARGS="-it" +else + # The input device on kokoro is not a TTY, so -it does not work. + DOCKER_TTY_ARGS= +fi + +# crosscompile protoc as we will later need it for the javascript build. +# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible +# with a wide range of linux distros (including any docker images we will use later to build and test javascript) +kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh + +# use an actual aarch64 docker image (with a real aarch64 nodejs) to run build & test protobuf javascript under an emulator +# * mount the protobuf root as /work to be able to access the crosscompiled files +# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force +# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user +# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, +# we just run map the user's home to a throwaway temporary directory +docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/node:16-buster kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh From 36243917c10fe4a25b05269353be36f039668fa2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 29 Apr 2021 16:46:12 +0200 Subject: [PATCH 2/3] add testing jobs for javascript_aarch64 --- kokoro/linux/javascript_aarch64/build.sh | 16 ++++++++++++++++ kokoro/linux/javascript_aarch64/continuous.cfg | 11 +++++++++++ kokoro/linux/javascript_aarch64/presubmit.cfg | 11 +++++++++++ 3 files changed, 38 insertions(+) create mode 100755 kokoro/linux/javascript_aarch64/build.sh create mode 100644 kokoro/linux/javascript_aarch64/continuous.cfg create mode 100644 kokoro/linux/javascript_aarch64/presubmit.cfg diff --git a/kokoro/linux/javascript_aarch64/build.sh b/kokoro/linux/javascript_aarch64/build.sh new file mode 100755 index 0000000000..dd9f8af864 --- /dev/null +++ b/kokoro/linux/javascript_aarch64/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "continuous" and "presubmit" jobs. + +set -ex + +# Change to repo root +cd $(dirname $0)/../../.. + +# Initialize any submodules. +git submodule update --init --recursive + +kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh + +kokoro/linux/aarch64/test_javascript_aarch64.sh diff --git a/kokoro/linux/javascript_aarch64/continuous.cfg b/kokoro/linux/javascript_aarch64/continuous.cfg new file mode 100644 index 0000000000..947bc759d6 --- /dev/null +++ b/kokoro/linux/javascript_aarch64/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript_aarch64/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javascript_aarch64/presubmit.cfg b/kokoro/linux/javascript_aarch64/presubmit.cfg new file mode 100644 index 0000000000..947bc759d6 --- /dev/null +++ b/kokoro/linux/javascript_aarch64/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript_aarch64/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} From f3411a29dabfc8a0b2d2919396f141015e72271c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 29 Apr 2021 19:00:14 +0200 Subject: [PATCH 3/3] node tests require java to be available --- .../javascript_build_and_run_tests_with_qemu_aarch64.sh | 9 +++++++++ kokoro/linux/aarch64/test_javascript_aarch64.sh | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh b/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh index e5d07e8bd2..c4eb7d8f0f 100755 --- a/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh +++ b/kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh @@ -2,6 +2,15 @@ set -ex +# install the same version of node as in /tests.sh +NODE_VERSION=node-v12.16.3-linux-arm64 +NODE_TGZ="$NODE_VERSION.tar.gz" +pushd /tmp +curl -OL https://nodejs.org/dist/v12.16.3/$NODE_TGZ +tar zxvf $NODE_TGZ +export PATH=$PATH:`pwd`/$NODE_VERSION/bin +popd + # go to the repo root cd $(dirname $0)/../../.. diff --git a/kokoro/linux/aarch64/test_javascript_aarch64.sh b/kokoro/linux/aarch64/test_javascript_aarch64.sh index dcddd60a2e..3156e6e9c1 100755 --- a/kokoro/linux/aarch64/test_javascript_aarch64.sh +++ b/kokoro/linux/aarch64/test_javascript_aarch64.sh @@ -23,4 +23,7 @@ kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh ko # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, # we just run map the user's home to a throwaway temporary directory -docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/node:16-buster kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh +# Note that the docker image used for running the tests is arm64v8/openjdk, not arm64v8/node +# This is because some of the node tests require java to be available and adding node +# binary distribution into a java image is easier than vice versa. +docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/openjdk:11-jdk-buster kokoro/linux/aarch64/javascript_build_and_run_tests_with_qemu_aarch64.sh