[Python 3.12] Deprecate distutil (#34186)

### Background

* `distutils` is deprecated with removal planned for Python 3.12
([pep-0632](https://peps.python.org/pep-0632/)), thus we're trying to
replace all distutils usage with setuptools.
* Please note that user still have access to `distutils` if setuptools
is installed and `SETUPTOOLS_USE_DISTUTILS` is set to `local` (The
default in setuptools, more details can be found [in this
discussion](https://github.com/pypa/setuptools/issues/2806#issuecomment-1193336591)).

### How we decide the replacement

* We're following setuptools [Porting from Distutils
guide](https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html#porting-from-distutils)
when deciding the replacement.

#### Replacement not mentioned in the guide

* Replaced `distutils.utils.get_platform()` with
`sysconfig.get_platform()`.
* Based on the [answer
here](https://stackoverflow.com/questions/71664875/what-is-the-replacement-for-distutils-util-get-platform),
and also checked the document that `sysconfig.get_platform()` is good
enough for our use cases.
* Replaced `DistutilsOptionError` with `OptionError`.
* `setuptools.error` is exporting it as `OptionError` [in the
code](https://github.com/pypa/setuptools/blob/v59.6.0/setuptools/errors.py).
* Upgrade `setuptools` in `test_packages.sh` and changed the version
ping to `59.6.0` in `build_artifact_python.bat`.
* `distutils.errors.*` is not fully re-exported until `59.0.0` (See
[this issue](https://github.com/pypa/setuptools/issues/2698) for more
details).

### Changes not included in this PR

* We're patching some compiler related functions provided by distutils
in our code
([example](ee4efc31c1/src/python/grpcio/_spawn_patch.py (L30))),
but since `setuptools` doesn't have similar interface (See [this issue
for more details](https://github.com/pypa/setuptools/issues/2806)), we
don't have a clear path to replace them yet.


<!--

If you know who should review your pull request, please assign it to
that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the
appropriate
lang label.

-->

setup.py

@@ -24,9 +24,6 @@ from distutils.unixccompiler import UnixCCompiler
 UnixCCompiler.src_extensions.append(".S")
 del UnixCCompiler
 
-from distutils import cygwinccompiler
-from distutils import extension as _extension
-from distutils import util
 import os
 import os.path
 import pathlib
@@ -41,6 +38,7 @@ import sysconfig
 
 import _metadata
 import pkg_resources
+from setuptools import Extension
 from setuptools.command import egg_info
 
 # Redirect the manifest template from MANIFEST.in to PYTHON-MANIFEST.in.
@@ -126,7 +124,7 @@ BUILD_WITH_BORING_SSL_ASM = _env_bool_value(
 
 # Export this environment variable to override the platform variant that will
 # be chosen for boringssl assembly optimizations. This option is useful when
-# crosscompiling and the host platform as obtained by distutils.utils.get_platform()
+# crosscompiling and the host platform as obtained by sysconfig.get_platform()
 # doesn't match the platform we are targetting.
 # Example value: "linux-aarch64"
 BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM = os.environ.get(
@@ -291,12 +289,6 @@ if EXTRA_ENV_LINK_ARGS is None:
         EXTRA_ENV_LINK_ARGS += " -lpthread"
         if check_linker_need_libatomic():
             EXTRA_ENV_LINK_ARGS += " -latomic"
-    elif "win32" in sys.platform and sys.version_info < (3, 5):
-        msvcr = cygwinccompiler.get_msvcr()[0]
-        EXTRA_ENV_LINK_ARGS += (
-            " -static-libgcc -static-libstdc++ -mcrtdll={msvcr}"
-            " -static -lshlwapi".format(msvcr=msvcr)
-        )
     if "linux" in sys.platform:
         EXTRA_ENV_LINK_ARGS += " -static-libgcc"
 
@@ -411,7 +403,7 @@ if BUILD_WITH_BORING_SSL_ASM and not BUILD_WITH_SYSTEM_OPENSSL:
     boringssl_asm_platform = (
         BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM
         if BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM
-        else util.get_platform()
+        else sysconfig.get_platform()
     )
     # BoringSSL's gas-compatible assembly files are all internally conditioned
     # by the preprocessor. Provided the platform has a gas-compatible assembler
@@ -490,7 +482,7 @@ if "darwin" in sys.platform:
             os.environ["_PYTHON_HOST_PLATFORM"] = re.sub(
                 r"macosx-[0-9]+\.[0-9]+-(.+)",
                 r"macosx-10.10-\1",
-                util.get_platform(),
+                sysconfig.get_platform(),
             )
 
 
@@ -513,7 +505,7 @@ def cython_extensions_and_necessity():
         core_c_files = list(CORE_C_FILES)
         extra_objects = []
     extensions = [
-        _extension.Extension(
+        Extension(
             name=module_name,
             sources=(
                 [module_file]

src/python/grpcio/_parallel_compile_patch.py

@@ -17,7 +17,6 @@ build_ext has lots of C/C++ files and normally them one by one.
 Enabling parallel build helps a lot.
 """
 
-import distutils.ccompiler
 import os
 
 try:
@@ -68,6 +67,12 @@ def _parallel_compile(
 
 
 def monkeypatch_compile_maybe():
-    """Monkeypatching is dumb, but the build speed gain is worth it."""
-    if BUILD_EXT_COMPILER_JOBS > 1:
+    """
+    Monkeypatching is dumb, but the build speed gain is worth it.
+    After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib.
+    """
+    use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "")
+    if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib":
+        import distutils.ccompiler  # pylint: disable=wrong-import-position
+
         distutils.ccompiler.CCompiler.compile = _parallel_compile

src/python/grpcio/commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 # NOTE(https://github.com/grpc/grpc/issues/24028): allow setuptools to monkey
 # patch distutils
@@ -185,7 +185,7 @@ def try_cythonize(extensions, linetracing=False, mandatory=True):
     """Attempt to cythonize the extensions.
 
     Args:
-      extensions: A list of `distutils.extension.Extension`.
+      extensions: A list of `setuptools.Extension`.
       linetracing: A bool indicating whether or not to enable linetracing.
       mandatory: Whether or not having Cython-generated files is mandatory. If it
         is, extensions will be poisoned when they can't be fully generated.

src/python/grpcio/support.py

@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from distutils import errors
 import os
 import os.path
 import shutil
 import sys
 import tempfile
 
+from setuptools import errors
+
 import commands
 
 C_PYTHON_DEV = """

src/python/grpcio_channelz/channelz_commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 import os
 import shutil

src/python/grpcio_health_checking/health_commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 import os
 import shutil

src/python/grpcio_reflection/reflection_commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 import os
 import shutil

src/python/grpcio_status/status_commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 import os
 import shutil

src/python/grpcio_testing/testing_commands.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the GRPC Python setup process."""
+"""Provides setuptools command classes for the GRPC Python setup process."""
 
 import os
 import shutil

src/python/grpcio_tests/commands.py

@@ -11,9 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Provides distutils command classes for the gRPC Python setup process."""
+"""Provides setuptools command classes for the gRPC Python setup process."""
 
-from distutils import errors as _errors
 import glob
 import os
 import os.path
@@ -23,6 +22,7 @@ import shutil
 import sys
 
 import setuptools
+from setuptools import errors as _errors
 from setuptools.command import build_ext
 from setuptools.command import build_py
 from setuptools.command import easy_install
@@ -305,7 +305,7 @@ class RunInterop(test.test):
 
     def finalize_options(self):
         if self.client and self.server:
-            raise _errors.DistutilsOptionError(
+            raise _errors.OptionError(
                 "you may only specify one of client or server"
             )
 

src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py

@@ -14,7 +14,6 @@
 
 import collections
 import contextlib
-import distutils.spawn
 import errno
 import itertools
 import os

test/distrib/python/test_packages.sh

@@ -37,7 +37,7 @@ TESTING_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-testing-[0-9]*.tar
 VIRTUAL_ENV=$(mktemp -d)
 python3 -m virtualenv "$VIRTUAL_ENV"
 PYTHON=$VIRTUAL_ENV/bin/python
-"$PYTHON" -m pip install --upgrade six pip wheel
+"$PYTHON" -m pip install --upgrade six pip wheel setuptools
 
 function validate_wheel_hashes() {
   for file in "$@"; do

tools/distrib/python/grpcio_tools/README.rst

@@ -141,7 +141,7 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory
 
   $ python -m grpc_tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES
 
-To use as a build step in distutils-based projects, you may use the provided
+To use as a build step in setuptools-based projects, you may use the provided
 command class in your :code:`setup.py`:
 
 ::
@@ -177,5 +177,4 @@ installed). One way to work around this can be found in our
 Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the
 command on-setup as desired.
 
-For more information on command classes, consult :code:`distutils` and
-:code:`setuptools` documentation.
+For more information on command classes, consult :code:`setuptools` documentation.

tools/distrib/python/grpcio_tools/_parallel_compile_patch.py

@@ -17,7 +17,6 @@ build_ext has lots of C/C++ files and normally them one by one.
 Enabling parallel build helps a lot.
 """
 
-import distutils.ccompiler
 import os
 
 try:
@@ -66,6 +65,12 @@ def _parallel_compile(
 
 
 def monkeypatch_compile_maybe():
-    """Monkeypatching is dumb, but the build speed gain is worth it."""
-    if BUILD_EXT_COMPILER_JOBS > 1:
+    """
+    Monkeypatching is dumb, but the build speed gain is worth it.
+    After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib.
+    """
+    use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "")
+    if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib":
+        import distutils.ccompiler  # pylint: disable=wrong-import-position
+
         distutils.ccompiler.CCompiler.compile = _parallel_compile

tools/distrib/python/grpcio_tools/setup.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from distutils import cygwinccompiler
-from distutils import extension
-from distutils import util
 import errno
 import os
 import os.path
@@ -29,6 +26,7 @@ import sysconfig
 
 import pkg_resources
 import setuptools
+from setuptools import Extension
 from setuptools.command import build_ext
 
 # TODO(atash) add flag to disable Cython use
@@ -191,12 +189,6 @@ if EXTRA_ENV_LINK_ARGS is None:
         EXTRA_ENV_LINK_ARGS += " -lpthread"
         if check_linker_need_libatomic():
             EXTRA_ENV_LINK_ARGS += " -latomic"
-    elif "win32" in sys.platform and sys.version_info < (3, 5):
-        msvcr = cygwinccompiler.get_msvcr()[0]
-        EXTRA_ENV_LINK_ARGS += (
-            " -static-libgcc -static-libstdc++ -mcrtdll={msvcr}"
-            " -static -lshlwapi".format(msvcr=msvcr)
-        )
 
 EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
 EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
@@ -228,7 +220,7 @@ if "win32" in sys.platform:
 elif "linux" in sys.platform or "darwin" in sys.platform:
     DEFINE_MACROS += (("HAVE_PTHREAD", 1),)
 
-# By default, Python3 distutils enforces compatibility of
+# By default, Python3 setuptools(distutils) enforces compatibility of
 # c plugins (.so files) with the OSX version Python was built with.
 # We need OSX 10.10, the oldest which supports C++ thread_local.
 if "darwin" in sys.platform:
@@ -241,7 +233,7 @@ if "darwin" in sys.platform:
         os.environ["_PYTHON_HOST_PLATFORM"] = re.sub(
             r"macosx-[0-9]+\.[0-9]+-(.+)",
             r"macosx-10.10-\1",
-            util.get_platform(),
+            sysconfig.get_platform(),
         )
 
 
@@ -281,7 +273,7 @@ def extension_modules():
         os.path.join("grpc_root", "src", "compiler", "proto_parser_helper.cc"),
     ] + CC_FILES
 
-    plugin_ext = extension.Extension(
+    plugin_ext = Extension(
         name="grpc_tools._protoc_compiler",
         sources=plugin_sources,
         include_dirs=[

tools/run_tests/artifacts/build_artifact_python.bat

@@ -21,7 +21,7 @@ set PATH=C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH%
 
 python -m pip install --upgrade six
 @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317
-python -m pip install --upgrade setuptools==44.1.1
+python -m pip install --upgrade setuptools==59.6.0
 python -m pip install --upgrade "cython<3.0.0rc1"
 python -m pip install -rrequirements.txt --user
 

tools/run_tests/artifacts/build_artifact_python.sh

@@ -26,7 +26,7 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc
 
 # Needed for building binary distribution wheels -- bdist_wheel
-"${PYTHON}" -m pip install --upgrade wheel
+"${PYTHON}" -m pip install --upgrade wheel setuptools
 
 if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ]
 then
@@ -182,7 +182,7 @@ fix_faulty_universal2_wheel() {
 }
 
 # This is necessary due to https://github.com/pypa/wheel/issues/406.
-# distutils incorrectly generates a universal2 artifact that only contains
+# wheel incorrectly generates a universal2 artifact that only contains
 # x86_64 libraries.
 if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then
   for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do

tools/run_tests/helper_scripts/build_python.sh

@@ -77,7 +77,7 @@ function venv_relative_python() {
   fi
 }
 
-# Distutils toolchain to use depending on the system.
+# Toolchain to use depending on the system.
 function toolchain() {
   if [ "$(is_mingw)" ]; then
     echo 'mingw32'
@@ -140,7 +140,6 @@ pip_install() {
 
 # Pin setuptools to < 60.0.0 to restore the distutil installation, see:
 # https://github.com/pypa/setuptools/pull/2896
-export SETUPTOOLS_USE_DISTUTILS=stdlib
 pip_install --upgrade pip==21.3.1
 pip_install --upgrade setuptools==59.6.0