From 4fa90f42ce5c7ecb61ec417582f06d0c9a20989b Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 7 Oct 2020 12:43:45 -0700 Subject: [PATCH 1/6] Support pre-compiled Python 3.9 wheels --- setup.py | 1 + src/python/grpcio_channelz/setup.py | 1 + src/python/grpcio_health_checking/setup.py | 1 + src/python/grpcio_reflection/setup.py | 1 + src/python/grpcio_status/setup.py | 1 + ...n38.ps1 => install_python_interpreters.ps1} | 18 ++++++++++++++++++ .../helper_scripts/prepare_build_macos_rc | 5 ++++- .../helper_scripts/prepare_build_windows.bat | 4 ++-- tools/run_tests/artifacts/artifact_targets.py | 5 +++++ 9 files changed, 34 insertions(+), 3 deletions(-) rename tools/internal_ci/helper_scripts/{install_python38.ps1 => install_python_interpreters.ps1} (84%) diff --git a/setup.py b/setup.py index bf013dd4944..184d5457f1a 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_channelz/setup.py b/src/python/grpcio_channelz/setup.py index f293ff9798f..678b5ef13f3 100644 --- a/src/python/grpcio_channelz/setup.py +++ b/src/python/grpcio_channelz/setup.py @@ -55,6 +55,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 62cb47004e4..fa1fbbd55ad 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -54,6 +54,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index ffb1177e3b9..2d0a3fcdaaf 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -55,6 +55,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', ] diff --git a/src/python/grpcio_status/setup.py b/src/python/grpcio_status/setup.py index bfedcabe00b..eb49069c34f 100644 --- a/src/python/grpcio_status/setup.py +++ b/src/python/grpcio_status/setup.py @@ -54,6 +54,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: Apache Software License', ] diff --git a/tools/internal_ci/helper_scripts/install_python38.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 similarity index 84% rename from tools/internal_ci/helper_scripts/install_python38.ps1 rename to tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index 26d8ed8bdc0..c2d46bab621 100644 --- a/tools/internal_ci/helper_scripts/install_python38.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -61,6 +61,7 @@ function Install-Python { Write-Host "Python $PythonVersion installed by $PythonInstaller at $PythonInstallPath." } +# Python 3.8 $Python38x86Config = @{ PythonVersion = "3.8.0" PythonInstaller = "python-3.8.0" @@ -76,3 +77,20 @@ $Python38x64Config = @{ PythonInstallerHash = "29ea87f24c32f5e924b7d63f8a08ee8d" } Install-Python @Python38x64Config + +# Python 3.9 +$Python39x86Config = @{ + PythonVersion = "3.9.0" + PythonInstaller = "python-3.9.0" + PythonInstallPath = "C:\Python39_32bit" + PythonInstallerHash = "4a2812db8ab9f2e522c96c7728cfcccb" +} +Install-Python @Python39x86Config + +$Python39x64Config = @{ + PythonVersion = "3.9.0" + PythonInstaller = "python-3.9.0-amd64" + PythonInstallPath = "C:\Python39" + PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63" +} +Install-Python @Python39x64Config diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 635f19596f2..47c2e31ba6f 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -94,14 +94,17 @@ then brew update-reset brew upgrade coreutils - # Install Python 3.7 and Python 3.8 + # Install Python 3.7, 3.8, 3.9 time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg + time curl -O https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.md5 echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5 + echo "16ca86fa3467e75bade26b8a9703c27f python-3.9.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5 md5sum -c /tmp/python_installer_checksum.md5 time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target / + time sudo installer -pkg ./python-3.9.0-macosx10.9.pkg -target / fi if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ] diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index f3b2fb6dde9..df2b724fbb7 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -36,10 +36,10 @@ python -m pip install google-api-python-client || goto :error powershell -File src\csharp\install_dotnet_sdk.ps1 || goto :error set PATH=%LOCALAPPDATA%\Microsoft\dotnet;%PATH% -@rem Install Python 3.8.0 +@rem Install Python interpreters @rem NOTE(lidiz): Python installer process may live longer than expected, and @rem has other side effects. It needs to be installed last to reduce impact. -powershell -File tools\internal_ci\helper_scripts\install_python38.ps1 || goto :error +powershell -File tools\internal_ci\helper_scripts\install_python_interpreters.ps1 || goto :error @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index fbf46b70374..d78c4223734 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -360,6 +360,7 @@ def targets(): PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), + PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'), PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), @@ -372,6 +373,7 @@ def targets(): PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), + PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'), PythonArtifact('linux_extra', 'armv7', '2.7'), PythonArtifact('linux_extra', 'armv7', '3.5'), PythonArtifact('linux_extra', 'armv7', '3.6'), @@ -383,16 +385,19 @@ def targets(): PythonArtifact('macos', 'x64', 'python3.6'), PythonArtifact('macos', 'x64', 'python3.7'), PythonArtifact('macos', 'x64', 'python3.8'), + PythonArtifact('macos', 'x64', 'python3.9'), PythonArtifact('windows', 'x86', 'Python27_32bit'), PythonArtifact('windows', 'x86', 'Python35_32bit'), PythonArtifact('windows', 'x86', 'Python36_32bit'), PythonArtifact('windows', 'x86', 'Python37_32bit'), PythonArtifact('windows', 'x86', 'Python38_32bit'), + PythonArtifact('windows', 'x86', 'Python39_32bit'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python37'), PythonArtifact('windows', 'x64', 'Python38'), + PythonArtifact('windows', 'x64', 'Python39'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64') From 81abd221d3904fb3a7707d732c169d1d6a98d371 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 7 Oct 2020 14:16:56 -0700 Subject: [PATCH 2/6] Try to see what's wrong --- .../internal_ci/helper_scripts/install_python_interpreters.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index c2d46bab621..387a9dfb6bd 100644 --- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -50,6 +50,9 @@ function Install-Python { Wait-Process -Name $PythonInstaller -Timeout 300 Write-Host "Installation process exits normally." + ls "C:\" + ls "$PythonInstallPath" + ls "$PythonInstallPath\python.exe" # Validate Python binary $PythonBinary = "$PythonInstallPath\python.exe" & $PythonBinary -c 'print(42)' From faad140178ce6c6f6a1966a0b899b3306f54872f Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 7 Oct 2020 15:44:49 -0700 Subject: [PATCH 3/6] Use /passive mode instead of /quite mode --- .../helper_scripts/install_python_interpreters.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index 387a9dfb6bd..9885adc34e1 100644 --- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -35,7 +35,7 @@ function Install-Python { Write-Host "Python installer $PythonInstallerPath validated." # Installs Python - & $PythonInstallerPath /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 TargetDir=$PythonInstallPath + & $PythonInstallerPath /passive InstallAllUsers=1 PrependPath=1 Include_test=0 TargetDir=$PythonInstallPath if (-Not $?) { throw "The Python installation exited with error!" } @@ -50,9 +50,6 @@ function Install-Python { Wait-Process -Name $PythonInstaller -Timeout 300 Write-Host "Installation process exits normally." - ls "C:\" - ls "$PythonInstallPath" - ls "$PythonInstallPath\python.exe" # Validate Python binary $PythonBinary = "$PythonInstallPath\python.exe" & $PythonBinary -c 'print(42)' From a910dc1802afc0be2f1976688a614f03dff55450 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Wed, 7 Oct 2020 16:47:54 -0700 Subject: [PATCH 4/6] Remove 3.9 Windows support --- .../helper_scripts/install_python_interpreters.ps1 | 4 ++-- tools/run_tests/artifacts/artifact_targets.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index 9885adc34e1..f00317518f9 100644 --- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -85,7 +85,7 @@ $Python39x86Config = @{ PythonInstallPath = "C:\Python39_32bit" PythonInstallerHash = "4a2812db8ab9f2e522c96c7728cfcccb" } -Install-Python @Python39x86Config +# Install-Python @Python39x86Config $Python39x64Config = @{ PythonVersion = "3.9.0" @@ -93,4 +93,4 @@ $Python39x64Config = @{ PythonInstallPath = "C:\Python39" PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63" } -Install-Python @Python39x64Config +# Install-Python @Python39x64Config diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index d78c4223734..3680e44cb5d 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -391,13 +391,15 @@ def targets(): PythonArtifact('windows', 'x86', 'Python36_32bit'), PythonArtifact('windows', 'x86', 'Python37_32bit'), PythonArtifact('windows', 'x86', 'Python38_32bit'), - PythonArtifact('windows', 'x86', 'Python39_32bit'), + # TODO(lidiz) uncomment if Python39 installs stably. + # PythonArtifact('windows', 'x86', 'Python39_32bit'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python37'), PythonArtifact('windows', 'x64', 'Python38'), - PythonArtifact('windows', 'x64', 'Python39'), + # TODO(lidiz) uncomment if Python39 installs stably. + # PythonArtifact('windows', 'x64', 'Python39'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64') From 9e2b8cee0785793e026525b2027e1989ef2cb83e Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 8 Oct 2020 13:54:07 -0700 Subject: [PATCH 5/6] Install Cython for Python 3.9 for artifact building jobs --- tools/internal_ci/macos/grpc_build_artifacts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 878d30dc12b..76ff43d3101 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -29,6 +29,7 @@ python3.5 -m pip install -U cython setuptools==44.1.1 wheel --user python3.6 -m pip install -U cython setuptools==44.1.1 wheel --user python3.7 -m pip install -U cython setuptools==44.1.1 wheel --user python3.8 -m pip install -U cython setuptools==44.1.1 wheel --user +python3.9 -m pip install -U cython setuptools==44.1.1 wheel --user # needed to build ruby artifacts time bash tools/distrib/build_ruby_environment_macos.sh From db16440066fec39c59afb233f7e394f8d2f0db9f Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 9 Oct 2020 15:31:11 -0700 Subject: [PATCH 6/6] Ensure virtualenv is installed in new interpreters --- tools/run_tests/artifacts/build_artifact_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index bf8b5ac6d95..7a342ec4ee3 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -137,6 +137,7 @@ then fi # Ensure the generated artifacts are valid. +"${PYTHON}" -m pip install virtualenv "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==16.7.9 && "${PYTHON}" -m virtualenv venv; } venv/bin/python -m pip install "twine<=2.0" venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/*