deannagarcia
4 years ago
committed by
GitHub
commit
e844a53716
58 changed files with 620 additions and 269 deletions
Binary file not shown.
@ -0,0 +1,26 @@ |
||||
#!/bin/bash |
||||
|
||||
set -e |
||||
|
||||
# go to the repo root |
||||
cd $(dirname $0)/../../../.. |
||||
|
||||
# running dockcross image without any arguments generates a wrapper |
||||
# scripts that can be used to run commands under the dockcross image |
||||
# easily. |
||||
# See https://github.com/dockcross/dockcross#usage for details |
||||
docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh |
||||
chmod +x dockcross-manylinux2014-aarch64.sh |
||||
|
||||
# the wrapper script has CRLF line endings and bash doesn't like that |
||||
# so we change CRLF line endings into LF. |
||||
sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh |
||||
|
||||
# The dockcross wrapper script runs arbitrary commands under the selected dockcross |
||||
# image with the following properties which make its use very convenient: |
||||
# * the current working directory is mounted under /work so the container can easily |
||||
# access the current workspace |
||||
# * the processes in the container run under the same UID and GID as the host process so unlike |
||||
# vanilla "docker run" invocations, the workspace doesn't get polluted with files |
||||
# owned by root. |
||||
./dockcross-manylinux2014-aarch64.sh "$@" |
||||
@ -0,0 +1,34 @@ |
||||
#!/bin/bash |
||||
# |
||||
# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator. |
||||
# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image. |
||||
|
||||
set -ex |
||||
|
||||
PYTHON="/opt/python/cp38-cp38/bin/python" |
||||
|
||||
./autogen.sh |
||||
CXXFLAGS="-fPIC -g -O2" ./configure |
||||
make -j8 |
||||
|
||||
pushd python |
||||
|
||||
# TODO: currently this step relies on qemu being registered with binfmt_misc so that |
||||
# aarch64 binaries are automatically run with an emulator. This works well once |
||||
# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine. |
||||
${PYTHON} setup.py build_py |
||||
|
||||
# when crosscompiling for aarch64, --plat-name needs to be set explicitly |
||||
# to end up with correctly named wheel file |
||||
# the value should be manylinuxABC_ARCH and dockcross docker image |
||||
# conveniently provides the value in the AUDITWHEEL_PLAT env |
||||
plat_name_flag="--plat-name=$AUDITWHEEL_PLAT" |
||||
|
||||
# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix |
||||
export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')" |
||||
|
||||
# Build the python extension inplace to be able to python unittests later |
||||
${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace |
||||
|
||||
# Build the binary wheel (to check it with auditwheel) |
||||
${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag |
||||
@ -0,0 +1,28 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
# go to the repo root |
||||
cd $(dirname $0)/../../.. |
||||
cd python |
||||
|
||||
PYTHON="/opt/python/cp38-cp38/bin/python" |
||||
${PYTHON} -m pip install --user six pytest auditwheel |
||||
|
||||
# check that we are really using aarch64 python |
||||
(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1) |
||||
|
||||
# step 1: run all python unittests |
||||
# we've built the python extension previously with --inplace option |
||||
# so we can just discover all the unittests and run them directly under |
||||
# the python/ directory. |
||||
LD_LIBRARY_PATH=../src/.libs PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf |
||||
|
||||
# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible. |
||||
# auditwheel needs to run on wheel's target platform (or under an emulator) |
||||
${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl |
||||
|
||||
# step 3: smoketest that the wheel can be installed and run a smokecheck |
||||
${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl |
||||
# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension |
||||
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")' |
||||
@ -0,0 +1,19 @@ |
||||
#!/bin/bash |
||||
|
||||
set -e |
||||
|
||||
# go to the repo root |
||||
cd $(dirname $0)/../../.. |
||||
|
||||
# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image |
||||
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh |
||||
|
||||
# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator |
||||
# * mount the protobuf root as /work to be able to access the crosscompiled files |
||||
# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution |
||||
# (manylinux_2_24 is debian-based while manylinux2014 is centos-based) |
||||
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force |
||||
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user |
||||
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, |
||||
# we just run map the user's home to a throwaway temporary directory |
||||
docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh |
||||
Loading…
Reference in new issue