From 2d7b751d4e4d07d525f2209a6caebcb94e6f75e8 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 20 Dec 2023 12:56:52 -0800 Subject: [PATCH] [Python run_test] Support Python3.12 (#35325) Fix: https://github.com/grpc/grpc/issues/35085 Closes #35325 PiperOrigin-RevId: 592635611 --- .../dockerfile/compile_python_312.include | 21 +++++++++ .../Dockerfile.template | 2 +- .../dockerimage_current_versions.bzl | 2 +- ...ython_debian11_default_x64.current_version | 2 +- .../python_debian11_default_x64/Dockerfile | 43 ++++++++++--------- .../macos/grpc_distribtests_python.sh | 1 + .../run_tests/helper_scripts/build_python.sh | 19 ++++---- tools/run_tests/run_tests.py | 15 +++++-- 8 files changed, 67 insertions(+), 38 deletions(-) create mode 100644 templates/tools/dockerfile/compile_python_312.include diff --git a/templates/tools/dockerfile/compile_python_312.include b/templates/tools/dockerfile/compile_python_312.include new file mode 100644 index 00000000000..d24e6bb66e9 --- /dev/null +++ b/templates/tools/dockerfile/compile_python_312.include @@ -0,0 +1,21 @@ +#================= +# Compile CPython 3.12.0 from source + +RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev libsqlite3-dev && apt-get clean +RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean + +RUN cd /tmp && ${'\\'} + wget -q https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz && ${'\\'} + tar xzvf Python-3.12.0.tgz && ${'\\'} + cd Python-3.12.0 && ${'\\'} + ./configure && ${'\\'} + make -j4 && ${'\\'} + make install + + +RUN cd /tmp && ${'\\'} + echo "d6eda3e1399cef5dfde7c4f319b0596c Python-3.12.0.tgz" > checksum.md5 && ${'\\'} + md5sum -c checksum.md5 + +RUN python3.12 -m ensurepip && ${'\\'} + python3.12 -m pip install coverage diff --git a/templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template index cb03c87c681..0981763f21d 100644 --- a/templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template @@ -19,11 +19,11 @@ <%include file="../../apt_get_basic.include"/> <%include file="../../run_tests_addons.include"/> - <%include file="../../compile_python_36.include"/> <%include file="../../compile_python_37.include"/> <%include file="../../compile_python_38.include"/> <%include file="../../compile_python_310.include"/> <%include file="../../compile_python_311.include"/> + <%include file="../../compile_python_312.include"/> # Python test coverage requires libsqlite3, and it have # to be installed before Python. diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index a9a845e802f..ce34a70136f 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -109,7 +109,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/test/php7_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_x64@sha256:302c06c5dbffb97dd5540d758a8ce849269527bb7d1c3885af0b956f8f33c49e", "tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:75fa424f2dae683422a5875d64911d9abf06c31e944401d240666d06f83de573", "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:fccca33a655c7aa89dd7ebd9492cbcc1f636bd2a004cd939d1982cfce3d68326", - "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:51107bfe5f9ef2c9d9c7a6554a16c4335a6d48f81a47cf6176fb36eca0605f02", + "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:a956ff4bfbfa4fc6a00c7113f2dada7ce8cf4011f236f07197732981875d6519", "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8", "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:7e77cf17e2e8657f4cc23ac9f93630bf13213fff961799e0f16dae17cd45cf6d", "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:e4cb502caccf2db733268ce2ddc951fda8a9df2f7f53d6b74523c33d40c83006", diff --git a/tools/dockerfile/test/python_debian11_default_x64.current_version b/tools/dockerfile/test/python_debian11_default_x64.current_version index 0e61fda7183..a318a5fcbc7 100644 --- a/tools/dockerfile/test/python_debian11_default_x64.current_version +++ b/tools/dockerfile/test/python_debian11_default_x64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:3aec01d65a59f4583b9a54369fc5d08b84b1017e@sha256:51107bfe5f9ef2c9d9c7a6554a16c4335a6d48f81a47cf6176fb36eca0605f02 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:05951fbd0e77e8ba072d270ad08f0d4b6ac00841@sha256:a956ff4bfbfa4fc6a00c7113f2dada7ce8cf4011f236f07197732981875d6519 \ No newline at end of file diff --git a/tools/dockerfile/test/python_debian11_default_x64/Dockerfile b/tools/dockerfile/test/python_debian11_default_x64/Dockerfile index aca43456da1..2ba17722bce 100644 --- a/tools/dockerfile/test/python_debian11_default_x64/Dockerfile +++ b/tools/dockerfile/test/python_debian11_default_x64/Dockerfile @@ -66,27 +66,6 @@ RUN git config --global protocol.file.allow always RUN mkdir /var/local/jenkins -#================= -# Compile CPython 3.6.15 from source - -RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev && apt-get clean -RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean - -RUN cd /tmp && \ - wget -q https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz && \ - tar xzvf Python-3.6.15.tgz && \ - cd Python-3.6.15 && \ - ./configure && \ - make -j4 && \ - make install - -RUN cd /tmp && \ - echo "f9e6f91c754a604f4fc6f6c7683723fb Python-3.6.15.tgz" > checksum.md5 && \ - md5sum -c checksum.md5 - -RUN python3.6 -m ensurepip && \ - python3.6 -m pip install coverage - #================= # Compile CPython 3.7.13 from source @@ -172,6 +151,28 @@ RUN cd /tmp && \ RUN python3.11 -m ensurepip && \ python3.11 -m pip install coverage +#================= +# Compile CPython 3.12.0 from source + +RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev libsqlite3-dev && apt-get clean +RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean + +RUN cd /tmp && \ + wget -q https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz && \ + tar xzvf Python-3.12.0.tgz && \ + cd Python-3.12.0 && \ + ./configure && \ + make -j4 && \ + make install + + +RUN cd /tmp && \ + echo "d6eda3e1399cef5dfde7c4f319b0596c Python-3.12.0.tgz" > checksum.md5 && \ + md5sum -c checksum.md5 + +RUN python3.12 -m ensurepip && \ + python3.12 -m pip install coverage + # Python test coverage requires libsqlite3, and it have # to be installed before Python. diff --git a/tools/internal_ci/macos/grpc_distribtests_python.sh b/tools/internal_ci/macos/grpc_distribtests_python.sh index 0361cae4ca3..ded41b34591 100644 --- a/tools/internal_ci/macos/grpc_distribtests_python.sh +++ b/tools/internal_ci/macos/grpc_distribtests_python.sh @@ -31,6 +31,7 @@ python3.8 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user python3.9 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user python3.10 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user python3.11 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user +python3.12 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user # Build all python macos artifacts (this step actually builds all the binary wheels and source archives) tools/run_tests/task_runner.py -f artifact macos python ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index e460d5b143f..61e05497870 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -123,23 +123,20 @@ if [[ "$(inside_venv)" ]]; then VENV_PYTHON="$PYTHON" else # Instantiate the virtualenv from the Python version passed in. - $PYTHON -m pip install --user virtualenv==20.0.23 - $PYTHON -m virtualenv "$VENV" + $PYTHON -m pip install --user virtualenv==20.25.0 + # Use --no-seed to prevent virtualenv from installing seed packages. + # Otherwise it might not find cython module while building grpcio. + $PYTHON -m virtualenv --no-seed "$VENV" VENV_PYTHON="$(pwd)/$VENV/$VENV_RELATIVE_PYTHON" fi - -# On library/version/platforms combo that do not have a binary -# published, we may end up building a dependency from source. In that -# case, several of our build environment variables may disrupt the -# third-party build process. This function pipes through only the -# minimal environment necessary. pip_install() { - /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install "$@" + $VENV_PYTHON -m pip install "$@" } -pip_install --upgrade setuptools==61.0.0 -pip_install --upgrade pip +$VENV_PYTHON -m ensurepip --upgrade +pip_install --upgrade wheel +pip_install --upgrade setuptools==66.1.0 # pip-installs the directory specified. Used because on MSYS the vanilla Windows # Python gets confused when parsing paths. diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0d679093215..0864ce5ea77 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -904,6 +904,13 @@ class PythonLanguage(object): bits=bits, config_vars=config_vars, ) + python312_config = _python_config_generator( + name="py312", + major="3", + minor="12", + bits=bits, + config_vars=config_vars, + ) pypy27_config = _pypy_config_generator( name="pypy", major="2", config_vars=config_vars ) @@ -928,7 +935,7 @@ class PythonLanguage(object): # Default set tested on master. Test oldest and newest. return ( python37_config, - python311_config, + python312_config, ) elif args.compiler == "python3.7": return (python37_config,) @@ -940,6 +947,8 @@ class PythonLanguage(object): return (python310_config,) elif args.compiler == "python3.11": return (python311_config,) + elif args.compiler == "python3.12": + return (python312_config,) elif args.compiler == "pypy": return (pypy27_config,) elif args.compiler == "pypy3": @@ -953,6 +962,7 @@ class PythonLanguage(object): python39_config, python310_config, python311_config, + python312_config, ) else: raise Exception("Compiler %s not supported." % args.compiler) @@ -1735,13 +1745,12 @@ argp.add_argument( "clang6", "clang16", # TODO: Automatically populate from supported version - "python2.7", - "python3.5", "python3.7", "python3.8", "python3.9", "python3.10", "python3.11", + "python3.12", "pypy", "pypy3", "python_alpine",