From 0088dae36a78b53686e7f4c847449e75e592a0e1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 17 Mar 2021 18:49:32 +0100 Subject: [PATCH] Use crosscompilation to build python armv7 wheels (#25704) * removed unused DOCKER_BASE_IMAGE functionality * remove legacy docker images for arm build * build python armv7 wheel via crosscompilation * add pyconfig.h hack * improve the dockerfile * yapf format code --- .../grpc_artifact_linux_armv6/Dockerfile | 26 ----------- .../grpc_artifact_linux_armv7/Dockerfile | 26 ----------- .../Dockerfile | 31 +++++++++++++ ...stall_python_for_wheel_crosscompilation.sh | 46 +++++++++++++++++++ tools/run_tests/artifacts/artifact_targets.py | 45 ++++++++---------- .../artifacts/build_artifact_python.sh | 16 +++++++ .../dockerize/build_and_run_docker.sh | 7 --- 7 files changed, 113 insertions(+), 84 deletions(-) delete mode 100644 tools/dockerfile/grpc_artifact_linux_armv6/Dockerfile delete mode 100644 tools/dockerfile/grpc_artifact_linux_armv7/Dockerfile create mode 100644 tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile create mode 100755 tools/dockerfile/grpc_artifact_python_linux_armv7/install_python_for_wheel_crosscompilation.sh diff --git a/tools/dockerfile/grpc_artifact_linux_armv6/Dockerfile b/tools/dockerfile/grpc_artifact_linux_armv6/Dockerfile deleted file mode 100644 index e2c406599dc..00000000000 --- a/tools/dockerfile/grpc_artifact_linux_armv6/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Docker file for building gRPC Raspbian binaries - -# TODO(https://github.com/grpc/grpc/issues/19199): Move off of this image. -FROM quay.io/grpc/raspbian_armv6 - -# Place any extra build instructions between these commands -# Recommend modifying upstream docker image (quay.io/grpc/raspbian_armv6) -# for build steps because running them under QEMU is very slow -# (https://github.com/kpayson64/armv7hf-debian-qemu) -RUN [ "cross-build-start" ] -RUN find /usr/local/bin -regex '.*python[0-9]+\.[0-9]+' | xargs -n1 -i{} bash -c "{} -m pip install --upgrade wheel setuptools" -RUN [ "cross-build-end" ] diff --git a/tools/dockerfile/grpc_artifact_linux_armv7/Dockerfile b/tools/dockerfile/grpc_artifact_linux_armv7/Dockerfile deleted file mode 100644 index 735c2e85b26..00000000000 --- a/tools/dockerfile/grpc_artifact_linux_armv7/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Docker file for building gRPC Raspbian binaries - -# TODO(https://github.com/grpc/grpc/issues/19199): Move off of this base image. -FROM quay.io/grpc/raspbian_armv7 - -# Place any extra build instructions between these commands -# Recommend modifying upstream docker image (quay.io/grpc/raspbian_armv7) -# for build steps because running them under QEMU is very slow -# (https://github.com/kpayson64/armv7hf-debian-qemu) -RUN [ "cross-build-start" ] -RUN find /usr/local/bin -regex '.*python[0-9]+\.[0-9]+' | xargs -n1 -i{} bash -c "{} -m pip install --upgrade wheel setuptools" -RUN [ "cross-build-end" ] diff --git a/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile new file mode 100644 index 00000000000..9af7874bbc1 --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile @@ -0,0 +1,31 @@ +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The aarch64 wheels are being crosscompiled to allow running the build +# on x64 machine. The dockcross/linux-armv7 image is a x86_64 +# image with crosscompilation toolchain installed +FROM dockcross/linux-armv7 + +RUN apt update && apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \ + libnss3-dev libssl-dev libreadline-dev libffi-dev && apt-get clean + +ADD install_python_for_wheel_crosscompilation.sh /scripts/install_python_for_wheel_crosscompilation.sh + +RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.6.13 /opt/python/cp36-cp36m +RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.7.10 /opt/python/cp37-cp37m +RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.8.8 /opt/python/cp38-cp38 +RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.9.2 /opt/python/cp39-cp39 + +ENV AUDITWHEEL_ARCH armv7l +ENV AUDITWHEEL_PLAT linux_armv7l diff --git a/tools/dockerfile/grpc_artifact_python_linux_armv7/install_python_for_wheel_crosscompilation.sh b/tools/dockerfile/grpc_artifact_python_linux_armv7/install_python_for_wheel_crosscompilation.sh new file mode 100755 index 00000000000..cc7349bb64c --- /dev/null +++ b/tools/dockerfile/grpc_artifact_python_linux_armv7/install_python_for_wheel_crosscompilation.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# ARGUMENTS +# $1 - python version in "3.X.Y" format +# $2 - python tags (as in manylinx images) e.g. "/opt/python/cp37-cp37m" +PYTHON_VERSION="$1" +PYTHON_PREFIX="$2" + +curl -O "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" +tar -xf "Python-${PYTHON_VERSION}.tar.xz" +pushd "Python-${PYTHON_VERSION}" + +# In this step, we are building python that can run on the architecture where the build runs (x64). +# Since the CC, CXX and other env vars are set by default to point to the crosscompilation toolchain, +# we explicitly unset them to end up with x64 python binaries. +(unset AS AR CC CPP CXX LD && ./configure --prefix="${PYTHON_PREFIX}" && make -j 4 && make install) + +# When building the crosscompiled native extension, python will automatically add its include directory +# that contains "Python.h" and other headers. But since we are going to be crosscompiling +# the wheels, we need the pyconfig.h that corresponds to the target architecture. +# Since pyconfig.h is the only generated header and python itself (once built) doesn't +# really need this header anymore, we simply generate a new pyconfig.h using our crosscompilation +# toolchain and overwrite the current ("wrong") version in the python's include directory. +./configure && cp pyconfig.h "${PYTHON_PREFIX}"/include/python* + +popd +# remove the build directory to decrease the overall docker image size +rm -rf "Python-${PYTHON_VERSION}" + +# install cython and wheel +"${PYTHON_PREFIX}/bin/pip3" install --upgrade cython wheel diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 2498ce17a60..7fb6c31bdc8 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -30,7 +30,6 @@ def create_docker_jobspec(name, flake_retries=0, timeout_retries=0, timeout_seconds=30 * 60, - docker_base_image=None, extra_docker_args=None, verbose_success=False): """Creates jobspec for a task running under docker.""" @@ -46,9 +45,6 @@ def create_docker_jobspec(name, 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 'OUTPUT_DIR': 'artifacts' } - - if docker_base_image is not None: - docker_env['DOCKER_BASE_IMAGE'] = docker_base_image if extra_docker_args is not None: docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args jobspec = jobset.JobSpec( @@ -118,24 +114,27 @@ class PythonArtifact: def build_jobspec(self): environ = {} if self.platform == 'linux_extra': - # Raspberry Pi build - environ['PYTHON'] = '/usr/local/bin/python{}'.format( + # Crosscompilation build for armv7 (e.g. Raspberry Pi) + environ['PYTHON'] = '/opt/python/{}/bin/python3'.format( self.py_version) - environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) - # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 - # A QEMU bug causes submodule update to freeze, so we copy directly - environ['RELATIVE_COPY_PATH'] = '.' - # Parallel builds are counterproductive in emulated environment - environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1' - extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' + environ['PIP'] = '/opt/python/{}/bin/pip3'.format(self.py_version) + environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE' + environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE' + # when crosscompiling, we need to force statically linking libstdc++ + # otherwise libstdc++ symbols would be too new and the resulting + # wheel wouldn't pass the auditwheel check. + # This is needed because C core won't build with GCC 4.8 that's + # included in the default dockcross toolchain and we needed + # to opt into using a slighly newer version of GCC. + environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE' + return create_docker_jobspec( self.name, - 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), + 'tools/dockerfile/grpc_artifact_python_linux_{}'.format( + self.arch), 'tools/run_tests/artifacts/build_artifact_python.sh', environ=environ, - timeout_seconds=60 * 60 * 7, - docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), - extra_docker_args=extra_args) + timeout_seconds=60 * 60) elif 'manylinux' in self.platform: if self.arch == 'x86': environ['SETARCH_CMD'] = 'linux32' @@ -165,8 +164,6 @@ class PythonArtifact: environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' return create_docker_jobspec( self.name, - # NOTE(rbellevi): Do *not* update this without also ensuring the - # base_docker_image attribute is accurate. 'tools/dockerfile/grpc_artifact_python_%s_%s' % (self.platform, self.arch), 'tools/run_tests/artifacts/build_artifact_python.sh', @@ -393,12 +390,10 @@ def targets(): PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m'), PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38'), PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'), - PythonArtifact('linux_extra', 'armv7', '2.7'), - PythonArtifact('linux_extra', 'armv7', '3.5'), - PythonArtifact('linux_extra', 'armv7', '3.6'), - PythonArtifact('linux_extra', 'armv6', '2.7'), - PythonArtifact('linux_extra', 'armv6', '3.5'), - PythonArtifact('linux_extra', 'armv6', '3.6'), + PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m'), + PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m'), + PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'), + PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'), PythonArtifact('macos', 'x64', 'python2.7'), PythonArtifact('macos', 'x64', 'python3.5'), PythonArtifact('macos', 'x64', 'python3.6'), diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 6615595cc33..7e5abcc4ab3 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -59,6 +59,22 @@ then export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-aarch64" fi +# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image. +if [ "$AUDITWHEEL_ARCH" == "armv7l" ] +then + # when crosscompiling for arm, --plat-name needs to be set explicitly + # to end up with correctly named wheel file + # our dockcross-based docker image onveniently provides the value in the AUDITWHEEL_PLAT env + WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT" + + # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix + GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-arm-linux-gnueabihf.so"))')" + export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX + + # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations + export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-arm" +fi + # Build the source distribution first because MANIFEST.in cannot override # exclusion of built shared objects among package resources (for some # inexplicable reason). diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 7e9bc7744eb..cc5cf1a1967 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -29,18 +29,11 @@ cd - # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) # OUTPUT_DIR - Directory that will be copied from inside docker after finishing. # DOCKERHUB_ORGANIZATION - If set, pull a prebuilt image from given dockerhub org. -# DOCKER_BASE_IMAGE - If set, pull the latest base image. # $@ - Extra args to pass to docker run # Use image name based on Dockerfile location checksum DOCKER_IMAGE_NAME=$(basename "$DOCKERFILE_DIR"):$(sha1sum "$DOCKERFILE_DIR/Dockerfile" | cut -f1 -d\ ) -# Pull the base image to force an update -if [ "$DOCKER_BASE_IMAGE" != "" ] -then - time docker pull "$DOCKER_BASE_IMAGE" -fi - if [ "$DOCKERHUB_ORGANIZATION" != "" ] then DOCKER_IMAGE_NAME=$DOCKERHUB_ORGANIZATION/$DOCKER_IMAGE_NAME