Use crosscompilation to build python armv7 wheels (#25704)
* removed unused DOCKER_BASE_IMAGE functionality * remove legacy docker images for arm build * build python armv7 wheel via crosscompilation * add pyconfig.h hack * improve the dockerfile * yapf format codepull/25745/head
Jan Tattermusch
4 years ago
committed by
GitHub
parent
d30b04ed7b
commit
0088dae36a
7 changed files with 113 additions and 84 deletions
@ -1,26 +0,0 @@ |
||||
# Copyright 2017 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Docker file for building gRPC Raspbian binaries |
||||
|
||||
# TODO(https://github.com/grpc/grpc/issues/19199): Move off of this image. |
||||
FROM quay.io/grpc/raspbian_armv6 |
||||
|
||||
# Place any extra build instructions between these commands |
||||
# Recommend modifying upstream docker image (quay.io/grpc/raspbian_armv6) |
||||
# for build steps because running them under QEMU is very slow |
||||
# (https://github.com/kpayson64/armv7hf-debian-qemu) |
||||
RUN [ "cross-build-start" ] |
||||
RUN find /usr/local/bin -regex '.*python[0-9]+\.[0-9]+' | xargs -n1 -i{} bash -c "{} -m pip install --upgrade wheel setuptools" |
||||
RUN [ "cross-build-end" ] |
||||
@ -1,26 +0,0 @@ |
||||
# Copyright 2017 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Docker file for building gRPC Raspbian binaries |
||||
|
||||
# TODO(https://github.com/grpc/grpc/issues/19199): Move off of this base image. |
||||
FROM quay.io/grpc/raspbian_armv7 |
||||
|
||||
# Place any extra build instructions between these commands |
||||
# Recommend modifying upstream docker image (quay.io/grpc/raspbian_armv7) |
||||
# for build steps because running them under QEMU is very slow |
||||
# (https://github.com/kpayson64/armv7hf-debian-qemu) |
||||
RUN [ "cross-build-start" ] |
||||
RUN find /usr/local/bin -regex '.*python[0-9]+\.[0-9]+' | xargs -n1 -i{} bash -c "{} -m pip install --upgrade wheel setuptools" |
||||
RUN [ "cross-build-end" ] |
||||
@ -0,0 +1,31 @@ |
||||
# Copyright 2020 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# The aarch64 wheels are being crosscompiled to allow running the build |
||||
# on x64 machine. The dockcross/linux-armv7 image is a x86_64 |
||||
# image with crosscompilation toolchain installed |
||||
FROM dockcross/linux-armv7 |
||||
|
||||
RUN apt update && apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \ |
||||
libnss3-dev libssl-dev libreadline-dev libffi-dev && apt-get clean |
||||
|
||||
ADD install_python_for_wheel_crosscompilation.sh /scripts/install_python_for_wheel_crosscompilation.sh |
||||
|
||||
RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.6.13 /opt/python/cp36-cp36m |
||||
RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.7.10 /opt/python/cp37-cp37m |
||||
RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.8.8 /opt/python/cp38-cp38 |
||||
RUN /scripts/install_python_for_wheel_crosscompilation.sh 3.9.2 /opt/python/cp39-cp39 |
||||
|
||||
ENV AUDITWHEEL_ARCH armv7l |
||||
ENV AUDITWHEEL_PLAT linux_armv7l |
||||
@ -0,0 +1,46 @@ |
||||
#!/bin/bash |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# ARGUMENTS |
||||
# $1 - python version in "3.X.Y" format |
||||
# $2 - python tags (as in manylinx images) e.g. "/opt/python/cp37-cp37m" |
||||
PYTHON_VERSION="$1" |
||||
PYTHON_PREFIX="$2" |
||||
|
||||
curl -O "https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz" |
||||
tar -xf "Python-${PYTHON_VERSION}.tar.xz" |
||||
pushd "Python-${PYTHON_VERSION}" |
||||
|
||||
# In this step, we are building python that can run on the architecture where the build runs (x64). |
||||
# Since the CC, CXX and other env vars are set by default to point to the crosscompilation toolchain, |
||||
# we explicitly unset them to end up with x64 python binaries. |
||||
(unset AS AR CC CPP CXX LD && ./configure --prefix="${PYTHON_PREFIX}" && make -j 4 && make install) |
||||
|
||||
# When building the crosscompiled native extension, python will automatically add its include directory |
||||
# that contains "Python.h" and other headers. But since we are going to be crosscompiling |
||||
# the wheels, we need the pyconfig.h that corresponds to the target architecture. |
||||
# Since pyconfig.h is the only generated header and python itself (once built) doesn't |
||||
# really need this header anymore, we simply generate a new pyconfig.h using our crosscompilation |
||||
# toolchain and overwrite the current ("wrong") version in the python's include directory. |
||||
./configure && cp pyconfig.h "${PYTHON_PREFIX}"/include/python* |
||||
|
||||
popd |
||||
# remove the build directory to decrease the overall docker image size |
||||
rm -rf "Python-${PYTHON_VERSION}" |
||||
|
||||
# install cython and wheel |
||||
"${PYTHON_PREFIX}/bin/pip3" install --upgrade cython wheel |
||||
Loading…
Reference in new issue