diff --git a/tools/internal_ci/linux/grpc_distribtests_python.sh b/tools/internal_ci/linux/grpc_distribtests_python.sh index aef918118ff..7192fcc707b 100755 --- a/tools/internal_ci/linux/grpc_distribtests_python.sh +++ b/tools/internal_ci/linux/grpc_distribtests_python.sh @@ -35,7 +35,7 @@ cp -r artifacts/* input_artifacts/ || true rm -rf artifacts_from_build_artifacts_step mv artifacts artifacts_from_build_artifacts_step || true -# This step mostly just copies artifacts from input_artifacts (but it also does some wheel stripping) +# This step only copies artifacts from input_artifacts tools/run_tests/task_runner.py -f package linux python -x build_packages/sponge_log.xml || FAILED="true" # the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 8677079f0ad..8f18eb38d8b 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -126,20 +126,41 @@ ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py sdist # shellcheck disable=SC2086 ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG +# run twine check before auditwheel, because auditwheel puts the repaired wheels into +# the artifacts output dir. +if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] +then + # Ensure the generated artifacts are valid. + # TODO(jtattermusch): avoid the need for always re-installing virtualenv and twine + "${PYTHON}" -m pip install virtualenv + "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==16.7.9 && "${PYTHON}" -m virtualenv venv; } + venv/bin/python -m pip install "twine<=2.0" + venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/* + rm -rf venv/ +fi + if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ] then for wheel in dist/*.whl; do "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" - "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" + "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" rm "$wheel" done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" - "${AUDITWHEEL}" repair "$wheel" -w "$ARTIFACT_DIR" + "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" rm "$wheel" done +else + cp -r dist/*.whl "$ARTIFACT_DIR" + cp -r tools/distrib/python/grpcio_tools/dist/*.whl "$ARTIFACT_DIR" fi +# grpcio and grpcio-tools wheels have already been copied to artifact_dir +# by "auditwheel repair", now copy the .tar.gz source archives as well. +cp -r dist/*.tar.gz "$ARTIFACT_DIR" +cp -r tools/distrib/python/grpcio_tools/dist/*.tar.gz "$ARTIFACT_DIR" + # We need to use the built grpcio-tools/grpcio to compile the health proto # Wheels are not supported by setup_requires/dependency_links, so we # manually install the dependency. Note we should only do this if we @@ -200,16 +221,3 @@ then sdist bdist_wheel cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR" fi - -if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] -then - # Ensure the generated artifacts are valid. - "${PYTHON}" -m pip install virtualenv - "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==16.7.9 && "${PYTHON}" -m virtualenv venv; } - venv/bin/python -m pip install "twine<=2.0" - venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/* - rm -rf venv/ -fi - -cp -r dist/* "$ARTIFACT_DIR" -cp -r tools/distrib/python/grpcio_tools/dist/* "$ARTIFACT_DIR" diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 9189cfa651b..29801a5b867 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -23,25 +23,6 @@ mkdir -p artifacts/ # and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true -export PYTHON=${PYTHON:-python} - -strip_binary_wheel() { - WHEEL_PATH="$1" - TEMP_WHEEL_DIR=$(mktemp -d) - ${PYTHON} -m wheel unpack "$WHEEL_PATH" -d "$TEMP_WHEEL_DIR" - find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" - find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" - - DISTRIBUTION_NAME=$(basename "$WHEEL_PATH" | cut -d '-' -f 1) - VERSION=$(basename "$WHEEL_PATH" | cut -d '-' -f 2) - ${PYTHON} -m wheel pack "$TEMP_WHEEL_DIR/$DISTRIBUTION_NAME-$VERSION" -d "$TEMP_WHEEL_DIR" - mv "${TEMP_WHEEL_DIR}"/*.whl "$WHEEL_PATH" -} - -for wheel in artifacts/*.whl; do - strip_binary_wheel "$wheel" -done - # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though.