From 16157cebe9b7da769b75665603c37332cb22d129 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 12 Nov 2020 12:31:17 -0800 Subject: [PATCH] Hopefully fixed 27 build on winserver2016 * Download the working msys64 and use it to compile 27 * Remove the cygwin detection override logic * Only install Python interpreters in jobs running on winserver2016 * Fix the batch script substring check * Increase the timeout for related jobs * TIL that there is an allow list for Kokoro env vars --- src/python/grpcio/commands.py | 5 +- .../install_python_interpreters.ps1 | 4 +- .../helper_scripts/prepare_build_windows.bat | 10 ++-- .../windows/grpc_build_artifacts.bat | 1 + .../windows/grpc_build_artifacts.cfg | 2 +- .../windows/grpc_run_tests_matrix.bat | 3 ++ .../pull_request/grpc_basictests_python.cfg | 4 +- tools/run_tests/artifacts/artifact_targets.py | 6 +-- .../artifacts/build_artifact_python.bat | 23 +++++++-- .../python_utils/download_and_unzip.py | 47 +++++++++++++++++++ 10 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 tools/run_tests/python_utils/download_and_unzip.py diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index aa031b1d8a1..a8b2ff5e612 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -218,8 +218,6 @@ class BuildExt(build_ext.build_ext): when invoked in C mode. GCC is okay with this, while clang is not. """ try: - if platform.system() == 'Windows': - return False # TODO(lidiz) Remove the generated a.out for success tests. cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'], stdin=subprocess.PIPE, @@ -228,7 +226,8 @@ class BuildExt(build_ext.build_ext): _, cc_err = cc_test.communicate(input=b'int main(){return 0;}') return not 'invalid argument' in str(cc_err) except: - sys.stderr.write(traceback.format_exc() + '\n') + sys.stderr.write('Non-fatal exception:' + + traceback.format_exc() + '\n') return False # This special conditioning is here due to difference of compiler diff --git a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index f00317518f9..9885adc34e1 100644 --- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -85,7 +85,7 @@ $Python39x86Config = @{ PythonInstallPath = "C:\Python39_32bit" PythonInstallerHash = "4a2812db8ab9f2e522c96c7728cfcccb" } -# Install-Python @Python39x86Config +Install-Python @Python39x86Config $Python39x64Config = @{ PythonVersion = "3.9.0" @@ -93,4 +93,4 @@ $Python39x64Config = @{ PythonInstallPath = "C:\Python39" PythonInstallerHash = "b61a33dc28f13b561452f3089c87eb63" } -# Install-Python @Python39x64Config +Install-Python @Python39x64Config diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index df2b724fbb7..fdc7b096420 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -36,16 +36,16 @@ python -m pip install google-api-python-client || goto :error powershell -File src\csharp\install_dotnet_sdk.ps1 || goto :error set PATH=%LOCALAPPDATA%\Microsoft\dotnet;%PATH% -@rem Install Python interpreters -@rem NOTE(lidiz): Python installer process may live longer than expected, and -@rem has other side effects. It needs to be installed last to reduce impact. -powershell -File tools\internal_ci\helper_scripts\install_python_interpreters.ps1 || goto :error - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true set DOTNET_CLI_TELEMETRY_OPTOUT=true +@rem Only install Python interpreters if we are running Python tests +If "%PREPARE_BUILD_INSTALL_DEPS_PYTHON%" == "true" ( + powershell -File tools\internal_ci\helper_scripts\install_python_interpreters.ps1 || goto :error +) + git submodule update --init || goto :error goto :EOF diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 473b6ee9822..b8901e384cf 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -19,6 +19,7 @@ choco install nasm -y --limit-output @rem enter repo root cd /d %~dp0\..\..\.. +set PREPARE_BUILD_INSTALL_DEPS_PYTHON=true call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1 python tools/run_tests/task_runner.py -f artifact windows -j 4 diff --git a/tools/internal_ci/windows/grpc_build_artifacts.cfg b/tools/internal_ci/windows/grpc_build_artifacts.cfg index f45cfda1215..909784939a9 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.cfg +++ b/tools/internal_ci/windows/grpc_build_artifacts.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat" -timeout_mins: 120 +timeout_mins: 180 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 01d1f811034..01d60262aac 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -15,6 +15,9 @@ @rem enter repo root cd /d %~dp0\..\..\.. +If Not "%RUN_TESTS_FLAGS%"=="%RUN_TESTS_FLAGS:python=%" ( + set PREPARE_BUILD_INSTALL_DEPS_PYTHON=true +) call tools/internal_ci/helper_scripts/prepare_build_windows.bat || exit /b 1 python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg index 6947e8e1b4e..aba512d17d9 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_python.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" -timeout_mins: 60 +timeout_mins: 90 action { define_artifacts { regex: "**/*sponge_log.*" @@ -26,5 +26,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests windows python -j 1 --inner_jobs 8 --internal_ci --max_time=3600" + value: "-f basictests windows python -j 1 --inner_jobs 8 --internal_ci --max_time=5400" } diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index f512d762f7e..fbbfcc80d93 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -400,15 +400,13 @@ def targets(): PythonArtifact('windows', 'x86', 'Python36_32bit'), PythonArtifact('windows', 'x86', 'Python37_32bit'), PythonArtifact('windows', 'x86', 'Python38_32bit'), - # TODO(lidiz) uncomment if Python39 installs stably. - # PythonArtifact('windows', 'x86', 'Python39_32bit'), + PythonArtifact('windows', 'x86', 'Python39_32bit'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python35'), PythonArtifact('windows', 'x64', 'Python36'), PythonArtifact('windows', 'x64', 'Python37'), PythonArtifact('windows', 'x64', 'Python38'), - # TODO(lidiz) uncomment if Python39 installs stably. - # PythonArtifact('windows', 'x64', 'Python39'), + PythonArtifact('windows', 'x64', 'Python39'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), PHPArtifact('linux', 'x64') diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index ff5a6af6fcd..b9b058bb28e 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -12,8 +12,14 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. -@rem set path to python & mingw compiler -set PATH=C:\%1;C:\%1\scripts;C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH% +@rem set path to python +set PATH=C:\%1;C:\%1\scripts;%PATH% + +if "%1" == "Python27" goto :install_mingw64_with_msvcr90 +if "%1" == "Python27_32bit" goto :install_mingw64_with_msvcr90 +@rem set path to the existed mingw compiler +set PATH=C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH% +:end_mingw64_installation python -m pip install --upgrade six @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 @@ -21,11 +27,12 @@ python -m pip install --upgrade setuptools==44.1.1 python -m pip install --upgrade cython python -m pip install -rrequirements.txt --user +@rem set GRPC_PYTHON_OVERRIDE_CYGWIN_DETECTION_FOR_27=1 set GRPC_PYTHON_BUILD_WITH_CYTHON=1 @rem Allow build_ext to build C/C++ files in parallel @rem by enabling a monkeypatch. It speeds up the build a lot. -set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2 +set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4 mkdir -p %ARTIFACTS_OUT% set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT% @@ -59,3 +66,13 @@ goto :EOF :error popd exit /b 1 + +:install_mingw64_with_msvcr90 +set MSYS64_DOWNLOAD_URL=https://storage.googleapis.com/grpc-build-helper/msys64.zip +set MSYS64_PATH=C:\tools\msys64_win7 +set PATH=%MSYS64_PATH%\mingw%2\bin;%PATH% +@rem Skip the installation if the directory exists +if exist "%MSYS64_PATH%" goto :end_mingw64_installation +python -m pip install requests || goto :error +python tools\run_tests\python_utils\download_and_unzip.py "%MSYS64_DOWNLOAD_URL%" "%MSYS64_PATH%" || goto :error +goto :end_mingw64_installation diff --git a/tools/run_tests/python_utils/download_and_unzip.py b/tools/run_tests/python_utils/download_and_unzip.py new file mode 100644 index 00000000000..e4b574fa3f1 --- /dev/null +++ b/tools/run_tests/python_utils/download_and_unzip.py @@ -0,0 +1,47 @@ +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Download and unzip the target file to the destination.""" + +from __future__ import print_function + +import os +import sys +import zipfile +import requests +import tempfile + + +def main(): + if len(sys.argv) != 3: + print("Usage: python download_and_unzip.py [zipfile-url] [destination]") + sys.exit(1) + download_url = sys.argv[1] + destination = sys.argv[2] + + with tempfile.TemporaryFile() as tmp_file: + r = requests.get(download_url) + if r.status_code != requests.codes.ok: + print("Download %s failed with [%d] \"%s\"" % + (download_url, r.status_code, r.text())) + sys.exit(1) + else: + tmp_file.write(r.content) + print("Successfully downloaded from %s", download_url) + with zipfile.ZipFile(tmp_file, 'r') as target_zip_file: + target_zip_file.extractall(destination) + print("Successfully unzip to %s" % destination) + + +if __name__ == "__main__": + main()