deannagarcia
4 years ago
committed by
GitHub
commit
e844a53716
58 changed files with 620 additions and 269 deletions
Binary file not shown.
@ -0,0 +1,26 @@ |
|||||||
|
#!/bin/bash |
||||||
|
|
||||||
|
set -e |
||||||
|
|
||||||
|
# go to the repo root |
||||||
|
cd $(dirname $0)/../../../.. |
||||||
|
|
||||||
|
# running dockcross image without any arguments generates a wrapper |
||||||
|
# scripts that can be used to run commands under the dockcross image |
||||||
|
# easily. |
||||||
|
# See https://github.com/dockcross/dockcross#usage for details |
||||||
|
docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh |
||||||
|
chmod +x dockcross-manylinux2014-aarch64.sh |
||||||
|
|
||||||
|
# the wrapper script has CRLF line endings and bash doesn't like that |
||||||
|
# so we change CRLF line endings into LF. |
||||||
|
sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh |
||||||
|
|
||||||
|
# The dockcross wrapper script runs arbitrary commands under the selected dockcross |
||||||
|
# image with the following properties which make its use very convenient: |
||||||
|
# * the current working directory is mounted under /work so the container can easily |
||||||
|
# access the current workspace |
||||||
|
# * the processes in the container run under the same UID and GID as the host process so unlike |
||||||
|
# vanilla "docker run" invocations, the workspace doesn't get polluted with files |
||||||
|
# owned by root. |
||||||
|
./dockcross-manylinux2014-aarch64.sh "$@" |
||||||
@ -0,0 +1,34 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# |
||||||
|
# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator. |
||||||
|
# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image. |
||||||
|
|
||||||
|
set -ex |
||||||
|
|
||||||
|
PYTHON="/opt/python/cp38-cp38/bin/python" |
||||||
|
|
||||||
|
./autogen.sh |
||||||
|
CXXFLAGS="-fPIC -g -O2" ./configure |
||||||
|
make -j8 |
||||||
|
|
||||||
|
pushd python |
||||||
|
|
||||||
|
# TODO: currently this step relies on qemu being registered with binfmt_misc so that |
||||||
|
# aarch64 binaries are automatically run with an emulator. This works well once |
||||||
|
# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine. |
||||||
|
${PYTHON} setup.py build_py |
||||||
|
|
||||||
|
# when crosscompiling for aarch64, --plat-name needs to be set explicitly |
||||||
|
# to end up with correctly named wheel file |
||||||
|
# the value should be manylinuxABC_ARCH and dockcross docker image |
||||||
|
# conveniently provides the value in the AUDITWHEEL_PLAT env |
||||||
|
plat_name_flag="--plat-name=$AUDITWHEEL_PLAT" |
||||||
|
|
||||||
|
# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix |
||||||
|
export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')" |
||||||
|
|
||||||
|
# Build the python extension inplace to be able to python unittests later |
||||||
|
${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace |
||||||
|
|
||||||
|
# Build the binary wheel (to check it with auditwheel) |
||||||
|
${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag |
||||||
@ -0,0 +1,28 @@ |
|||||||
|
#!/bin/bash |
||||||
|
|
||||||
|
set -ex |
||||||
|
|
||||||
|
# go to the repo root |
||||||
|
cd $(dirname $0)/../../.. |
||||||
|
cd python |
||||||
|
|
||||||
|
PYTHON="/opt/python/cp38-cp38/bin/python" |
||||||
|
${PYTHON} -m pip install --user six pytest auditwheel |
||||||
|
|
||||||
|
# check that we are really using aarch64 python |
||||||
|
(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1) |
||||||
|
|
||||||
|
# step 1: run all python unittests |
||||||
|
# we've built the python extension previously with --inplace option |
||||||
|
# so we can just discover all the unittests and run them directly under |
||||||
|
# the python/ directory. |
||||||
|
LD_LIBRARY_PATH=../src/.libs PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf |
||||||
|
|
||||||
|
# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible. |
||||||
|
# auditwheel needs to run on wheel's target platform (or under an emulator) |
||||||
|
${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl |
||||||
|
|
||||||
|
# step 3: smoketest that the wheel can be installed and run a smokecheck |
||||||
|
${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl |
||||||
|
# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension |
||||||
|
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")' |
||||||
@ -0,0 +1,19 @@ |
|||||||
|
#!/bin/bash |
||||||
|
|
||||||
|
set -e |
||||||
|
|
||||||
|
# go to the repo root |
||||||
|
cd $(dirname $0)/../../.. |
||||||
|
|
||||||
|
# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image |
||||||
|
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh |
||||||
|
|
||||||
|
# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator |
||||||
|
# * mount the protobuf root as /work to be able to access the crosscompiled files |
||||||
|
# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution |
||||||
|
# (manylinux_2_24 is debian-based while manylinux2014 is centos-based) |
||||||
|
# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force |
||||||
|
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user |
||||||
|
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, |
||||||
|
# we just run map the user's home to a throwaway temporary directory |
||||||
|
docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh |
||||||
Loading…
Reference in new issue