From 127a6c1d9e3eb17b66929eedd1cfef73ebe72706 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 8 Mar 2019 10:29:00 -0800 Subject: [PATCH 1/3] Revert "Revert "Roll foward "Strip Python wheel binary""" This reverts commit 98fc9022005126897b809c5337944afadf27c619. --- test/distrib/python/test_packages.sh | 2 +- .../artifacts/build_package_python.sh | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/test/distrib/python/test_packages.sh b/test/distrib/python/test_packages.sh index 755daa10211..433148e6bd7 100755 --- a/test/distrib/python/test_packages.sh +++ b/test/distrib/python/test_packages.sh @@ -41,7 +41,7 @@ PYTHON=$VIRTUAL_ENV/bin/python function at_least_one_installs() { for file in "$@"; do - if "$PYTHON" -m pip install "$file"; then + if "$PYTHON" -m pip install --require-hashes "$file"; then return 0 fi done diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29801a5b867..29a26bc081c 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -23,6 +23,27 @@ mkdir -p artifacts/ # and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true +apt-get install -y python-pip +python -m pip install wheel --user + +strip_binary_wheel() { + WHEEL_PATH="$1" + TEMP_WHEEL_DIR=$(mktemp -d) + python -m wheel unpack "$WHEEL_PATH" -d "$TEMP_WHEEL_DIR" + find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" + find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" + + WHEEL_FILE=$(basename "$WHEEL_PATH") + DISTRIBUTION_NAME=$(basename "$WHEEL_PATH" | cut -d '-' -f 1) + VERSION=$(basename "$WHEEL_PATH" | cut -d '-' -f 2) + python -m wheel pack "$TEMP_WHEEL_DIR/$DISTRIBUTION_NAME-$VERSION" -d "$TEMP_WHEEL_DIR" + mv "$TEMP_WHEEL_DIR/$WHEEL_FILE" "$WHEEL_PATH" +} + +for wheel in artifacts/*.whl; do + strip_binary_wheel "$wheel" +done + # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. From cb9cc5592e3259c60f8c7f7f3d30133a322ebaba Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 7 Mar 2019 17:59:39 -0800 Subject: [PATCH 2/3] Upgrade pip before using wheel --- tools/run_tests/artifacts/build_package_python.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29a26bc081c..35c78e9a93e 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -24,7 +24,8 @@ mkdir -p artifacts/ cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true apt-get install -y python-pip -python -m pip install wheel --user +python -m pip install -U pip +python -m pip install -U wheel strip_binary_wheel() { WHEEL_PATH="$1" From fa7f01079517cf7075d3830eb8a207aacaa23ec6 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 8 Mar 2019 10:47:04 -0800 Subject: [PATCH 3/3] Fix the hash checking mechanism --- test/distrib/python/test_packages.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/distrib/python/test_packages.sh b/test/distrib/python/test_packages.sh index 433148e6bd7..4e1e6dbc94f 100755 --- a/test/distrib/python/test_packages.sh +++ b/test/distrib/python/test_packages.sh @@ -39,9 +39,16 @@ virtualenv "$VIRTUAL_ENV" PYTHON=$VIRTUAL_ENV/bin/python "$PYTHON" -m pip install --upgrade six pip +function validate_wheel_hashes() { + for file in "$@"; do + "$PYTHON" -m wheel unpack "$file" --dest-dir /tmp || return 1 + done + return 0 +} + function at_least_one_installs() { for file in "$@"; do - if "$PYTHON" -m pip install --require-hashes "$file"; then + if "$PYTHON" -m pip install "$file"; then return 0 fi done @@ -49,6 +56,16 @@ function at_least_one_installs() { } +# +# Validate the files in wheel matches their hashes and size in RECORD +# + +if [[ "$1" == "binary" ]]; then + validate_wheel_hashes "${ARCHIVES[@]}" + validate_wheel_hashes "${TOOLS_ARCHIVES[@]}" +fi + + # # Install our distributions in order of dependencies #