[Python run_test] Support Python3.12 (#35325)

Fix: https://github.com/grpc/grpc/issues/35085
<!--

If you know who should review your pull request, please assign it to that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the appropriate
lang label.

-->

Closes #35325

PiperOrigin-RevId: 592635611

templates/tools/dockerfile/compile_python_312.include

@@ -0,0 +1,21 @@
+#=================
+# Compile CPython 3.12.0 from source
+
+RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev libsqlite3-dev && apt-get clean
+RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean
+
+RUN cd /tmp && ${'\\'}
+    wget -q https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz && ${'\\'}
+    tar xzvf Python-3.12.0.tgz && ${'\\'}
+    cd Python-3.12.0 && ${'\\'}
+    ./configure && ${'\\'}
+    make -j4 && ${'\\'}
+    make install
+
+
+RUN cd /tmp && ${'\\'}
+    echo "d6eda3e1399cef5dfde7c4f319b0596c  Python-3.12.0.tgz" > checksum.md5 && ${'\\'}
+    md5sum -c checksum.md5
+
+RUN python3.12 -m ensurepip && ${'\\'}
+    python3.12 -m pip install coverage

templates/tools/dockerfile/test/python_debian11_default_x64/Dockerfile.template

@@ -19,11 +19,11 @@
   <%include file="../../apt_get_basic.include"/>
   <%include file="../../run_tests_addons.include"/>
 
-  <%include file="../../compile_python_36.include"/>
   <%include file="../../compile_python_37.include"/>
   <%include file="../../compile_python_38.include"/>
   <%include file="../../compile_python_310.include"/>
   <%include file="../../compile_python_311.include"/>
+  <%include file="../../compile_python_312.include"/>
 
   # Python test coverage requires libsqlite3, and it have
   # to be installed before Python.

tools/bazelify_tests/dockerimage_current_versions.bzl

@@ -109,7 +109,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = {
     "tools/dockerfile/test/php7_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_x64@sha256:302c06c5dbffb97dd5540d758a8ce849269527bb7d1c3885af0b956f8f33c49e",
     "tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:75fa424f2dae683422a5875d64911d9abf06c31e944401d240666d06f83de573",
     "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:fccca33a655c7aa89dd7ebd9492cbcc1f636bd2a004cd939d1982cfce3d68326",
-    "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:51107bfe5f9ef2c9d9c7a6554a16c4335a6d48f81a47cf6176fb36eca0605f02",
+    "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:a956ff4bfbfa4fc6a00c7113f2dada7ce8cf4011f236f07197732981875d6519",
     "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8",
     "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:7e77cf17e2e8657f4cc23ac9f93630bf13213fff961799e0f16dae17cd45cf6d",
     "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:e4cb502caccf2db733268ce2ddc951fda8a9df2f7f53d6b74523c33d40c83006",

tools/dockerfile/test/python_debian11_default_x64.current_version

@@ -1 +1 @@
-us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:3aec01d65a59f4583b9a54369fc5d08b84b1017e@sha256:51107bfe5f9ef2c9d9c7a6554a16c4335a6d48f81a47cf6176fb36eca0605f02
+us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64:05951fbd0e77e8ba072d270ad08f0d4b6ac00841@sha256:a956ff4bfbfa4fc6a00c7113f2dada7ce8cf4011f236f07197732981875d6519

tools/dockerfile/test/python_debian11_default_x64/Dockerfile

@@ -66,27 +66,6 @@ RUN git config --global protocol.file.allow always
 RUN mkdir /var/local/jenkins
 
 
-#=================
-# Compile CPython 3.6.15 from source
-
-RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev && apt-get clean
-RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean
-
-RUN cd /tmp && \
-    wget -q https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz && \
-    tar xzvf Python-3.6.15.tgz && \
-    cd Python-3.6.15 && \
-    ./configure && \
-    make -j4 && \
-    make install
-
-RUN cd /tmp && \
-    echo "f9e6f91c754a604f4fc6f6c7683723fb Python-3.6.15.tgz" > checksum.md5 && \
-    md5sum -c checksum.md5
-
-RUN python3.6 -m ensurepip && \
-    python3.6 -m pip install coverage
-
 #=================
 # Compile CPython 3.7.13 from source
 
@@ -172,6 +151,28 @@ RUN cd /tmp && \
 RUN python3.11 -m ensurepip && \
     python3.11 -m pip install coverage
 
+#=================
+# Compile CPython 3.12.0 from source
+
+RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev libsqlite3-dev && apt-get clean
+RUN apt-get update && apt-get install -y jq build-essential libffi-dev && apt-get clean
+
+RUN cd /tmp && \
+    wget -q https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz && \
+    tar xzvf Python-3.12.0.tgz && \
+    cd Python-3.12.0 && \
+    ./configure && \
+    make -j4 && \
+    make install
+
+
+RUN cd /tmp && \
+    echo "d6eda3e1399cef5dfde7c4f319b0596c  Python-3.12.0.tgz" > checksum.md5 && \
+    md5sum -c checksum.md5
+
+RUN python3.12 -m ensurepip && \
+    python3.12 -m pip install coverage
+
 
 # Python test coverage requires libsqlite3, and it have
 # to be installed before Python.

tools/internal_ci/macos/grpc_distribtests_python.sh

@@ -31,6 +31,7 @@ python3.8 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user
 python3.9 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user
 python3.10 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user
 python3.11 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user
+python3.12 -m pip install -U 'cython<3.0.0rc1' setuptools==65.4.1 wheel --user
 
 # Build all python macos artifacts (this step actually builds all the binary wheels and source archives)
 tools/run_tests/task_runner.py -f artifact macos python ${TASK_RUNNER_EXTRA_FILTERS} -j 4 -x build_artifacts/sponge_log.xml || FAILED="true"

tools/run_tests/helper_scripts/build_python.sh

@@ -123,23 +123,20 @@ if [[ "$(inside_venv)" ]]; then
   VENV_PYTHON="$PYTHON"
 else
   # Instantiate the virtualenv from the Python version passed in.
-  $PYTHON -m pip install --user virtualenv==20.0.23
-  $PYTHON -m virtualenv "$VENV"
+  $PYTHON -m pip install --user virtualenv==20.25.0
+  # Use --no-seed to prevent virtualenv from installing seed packages.
+  # Otherwise it might not find cython module while building grpcio.
+  $PYTHON -m virtualenv --no-seed "$VENV"
   VENV_PYTHON="$(pwd)/$VENV/$VENV_RELATIVE_PYTHON"
 fi
 
-
-# On library/version/platforms combo that do not have a binary
-# published, we may end up building a dependency from source. In that
-# case, several of our build environment variables may disrupt the
-# third-party build process. This function pipes through only the
-# minimal environment necessary.
 pip_install() {
-  /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install "$@"
+  $VENV_PYTHON -m pip install "$@"
 }
 
-pip_install --upgrade setuptools==61.0.0
-pip_install --upgrade pip
+$VENV_PYTHON -m ensurepip --upgrade
+pip_install --upgrade wheel
+pip_install --upgrade setuptools==66.1.0
 
 # pip-installs the directory specified. Used because on MSYS the vanilla Windows
 # Python gets confused when parsing paths.

tools/run_tests/run_tests.py

@@ -904,6 +904,13 @@ class PythonLanguage(object):
             bits=bits,
             config_vars=config_vars,
         )
+        python312_config = _python_config_generator(
+            name="py312",
+            major="3",
+            minor="12",
+            bits=bits,
+            config_vars=config_vars,
+        )
         pypy27_config = _pypy_config_generator(
             name="pypy", major="2", config_vars=config_vars
         )
@@ -928,7 +935,7 @@ class PythonLanguage(object):
                 # Default set tested on master. Test oldest and newest.
                 return (
                     python37_config,
-                    python311_config,
+                    python312_config,
                 )
         elif args.compiler == "python3.7":
             return (python37_config,)
@@ -940,6 +947,8 @@ class PythonLanguage(object):
             return (python310_config,)
         elif args.compiler == "python3.11":
             return (python311_config,)
+        elif args.compiler == "python3.12":
+            return (python312_config,)
         elif args.compiler == "pypy":
             return (pypy27_config,)
         elif args.compiler == "pypy3":
@@ -953,6 +962,7 @@ class PythonLanguage(object):
                 python39_config,
                 python310_config,
                 python311_config,
+                python312_config,
             )
         else:
             raise Exception("Compiler %s not supported." % args.compiler)
@@ -1735,13 +1745,12 @@ argp.add_argument(
         "clang6",
         "clang16",
         # TODO: Automatically populate from supported version
-        "python2.7",
-        "python3.5",
         "python3.7",
         "python3.8",
         "python3.9",
         "python3.10",
         "python3.11",
+        "python3.12",
         "pypy",
         "pypy3",
         "python_alpine",