From c7a13365665ed9f716e799ffd57d8d2716b34775 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Wed, 6 Sep 2023 14:32:30 -0700 Subject: [PATCH] [Python 3.12] Deprecate distutil (#34186) ### Background * `distutils` is deprecated with removal planned for Python 3.12 ([pep-0632](https://peps.python.org/pep-0632/)), thus we're trying to replace all distutils usage with setuptools. * Please note that user still have access to `distutils` if setuptools is installed and `SETUPTOOLS_USE_DISTUTILS` is set to `local` (The default in setuptools, more details can be found [in this discussion](https://github.com/pypa/setuptools/issues/2806#issuecomment-1193336591)). ### How we decide the replacement * We're following setuptools [Porting from Distutils guide](https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html#porting-from-distutils) when deciding the replacement. #### Replacement not mentioned in the guide * Replaced `distutils.utils.get_platform()` with `sysconfig.get_platform()`. * Based on the [answer here](https://stackoverflow.com/questions/71664875/what-is-the-replacement-for-distutils-util-get-platform), and also checked the document that `sysconfig.get_platform()` is good enough for our use cases. * Replaced `DistutilsOptionError` with `OptionError`. * `setuptools.error` is exporting it as `OptionError` [in the code](https://github.com/pypa/setuptools/blob/v59.6.0/setuptools/errors.py). * Upgrade `setuptools` in `test_packages.sh` and changed the version ping to `59.6.0` in `build_artifact_python.bat`. * `distutils.errors.*` is not fully re-exported until `59.0.0` (See [this issue](https://github.com/pypa/setuptools/issues/2698) for more details). ### Changes not included in this PR * We're patching some compiler related functions provided by distutils in our code ([example](https://github.com/grpc/grpc/blob/ee4efc31c1dde7389ece70ba908049d7baeb9c65/src/python/grpcio/_spawn_patch.py#L30)), but since `setuptools` doesn't have similar interface (See [this issue for more details](https://github.com/pypa/setuptools/issues/2806)), we don't have a clear path to replace them yet. --- setup.py | 18 +++++------------- src/python/grpcio/_parallel_compile_patch.py | 11 ++++++++--- src/python/grpcio/commands.py | 4 ++-- src/python/grpcio/support.py | 3 ++- .../grpcio_channelz/channelz_commands.py | 2 +- .../grpcio_health_checking/health_commands.py | 2 +- .../grpcio_reflection/reflection_commands.py | 2 +- src/python/grpcio_status/status_commands.py | 2 +- src/python/grpcio_testing/testing_commands.py | 2 +- src/python/grpcio_tests/commands.py | 6 +++--- .../tests/protoc_plugin/_python_plugin_test.py | 1 - test/distrib/python/test_packages.sh | 2 +- tools/distrib/python/grpcio_tools/README.rst | 5 ++--- .../grpcio_tools/_parallel_compile_patch.py | 11 ++++++++--- tools/distrib/python/grpcio_tools/setup.py | 16 ++++------------ .../artifacts/build_artifact_python.bat | 2 +- .../artifacts/build_artifact_python.sh | 4 ++-- tools/run_tests/helper_scripts/build_python.sh | 3 +-- 18 files changed, 44 insertions(+), 52 deletions(-) diff --git a/setup.py b/setup.py index 4e4d9cfe4d6..f42f459df87 100644 --- a/setup.py +++ b/setup.py @@ -24,9 +24,6 @@ from distutils.unixccompiler import UnixCCompiler UnixCCompiler.src_extensions.append(".S") del UnixCCompiler -from distutils import cygwinccompiler -from distutils import extension as _extension -from distutils import util import os import os.path import pathlib @@ -41,6 +38,7 @@ import sysconfig import _metadata import pkg_resources +from setuptools import Extension from setuptools.command import egg_info # Redirect the manifest template from MANIFEST.in to PYTHON-MANIFEST.in. @@ -126,7 +124,7 @@ BUILD_WITH_BORING_SSL_ASM = _env_bool_value( # Export this environment variable to override the platform variant that will # be chosen for boringssl assembly optimizations. This option is useful when -# crosscompiling and the host platform as obtained by distutils.utils.get_platform() +# crosscompiling and the host platform as obtained by sysconfig.get_platform() # doesn't match the platform we are targetting. # Example value: "linux-aarch64" BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM = os.environ.get( @@ -291,12 +289,6 @@ if EXTRA_ENV_LINK_ARGS is None: EXTRA_ENV_LINK_ARGS += " -lpthread" if check_linker_need_libatomic(): EXTRA_ENV_LINK_ARGS += " -latomic" - elif "win32" in sys.platform and sys.version_info < (3, 5): - msvcr = cygwinccompiler.get_msvcr()[0] - EXTRA_ENV_LINK_ARGS += ( - " -static-libgcc -static-libstdc++ -mcrtdll={msvcr}" - " -static -lshlwapi".format(msvcr=msvcr) - ) if "linux" in sys.platform: EXTRA_ENV_LINK_ARGS += " -static-libgcc" @@ -411,7 +403,7 @@ if BUILD_WITH_BORING_SSL_ASM and not BUILD_WITH_SYSTEM_OPENSSL: boringssl_asm_platform = ( BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM if BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM - else util.get_platform() + else sysconfig.get_platform() ) # BoringSSL's gas-compatible assembly files are all internally conditioned # by the preprocessor. Provided the platform has a gas-compatible assembler @@ -490,7 +482,7 @@ if "darwin" in sys.platform: os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( r"macosx-[0-9]+\.[0-9]+-(.+)", r"macosx-10.10-\1", - util.get_platform(), + sysconfig.get_platform(), ) @@ -513,7 +505,7 @@ def cython_extensions_and_necessity(): core_c_files = list(CORE_C_FILES) extra_objects = [] extensions = [ - _extension.Extension( + Extension( name=module_name, sources=( [module_file] diff --git a/src/python/grpcio/_parallel_compile_patch.py b/src/python/grpcio/_parallel_compile_patch.py index 4adc3630b5a..de095e658c9 100644 --- a/src/python/grpcio/_parallel_compile_patch.py +++ b/src/python/grpcio/_parallel_compile_patch.py @@ -17,7 +17,6 @@ build_ext has lots of C/C++ files and normally them one by one. Enabling parallel build helps a lot. """ -import distutils.ccompiler import os try: @@ -68,6 +67,12 @@ def _parallel_compile( def monkeypatch_compile_maybe(): - """Monkeypatching is dumb, but the build speed gain is worth it.""" - if BUILD_EXT_COMPILER_JOBS > 1: + """ + Monkeypatching is dumb, but the build speed gain is worth it. + After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib. + """ + use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "") + if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib": + import distutils.ccompiler # pylint: disable=wrong-import-position + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 3739097a745..ae37f777aa1 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" # NOTE(https://github.com/grpc/grpc/issues/24028): allow setuptools to monkey # patch distutils @@ -185,7 +185,7 @@ def try_cythonize(extensions, linetracing=False, mandatory=True): """Attempt to cythonize the extensions. Args: - extensions: A list of `distutils.extension.Extension`. + extensions: A list of `setuptools.Extension`. linetracing: A bool indicating whether or not to enable linetracing. mandatory: Whether or not having Cython-generated files is mandatory. If it is, extensions will be poisoned when they can't be fully generated. diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py index 6f0a59df7c1..cab3be17e87 100644 --- a/src/python/grpcio/support.py +++ b/src/python/grpcio/support.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from distutils import errors import os import os.path import shutil import sys import tempfile +from setuptools import errors + import commands C_PYTHON_DEV = """ diff --git a/src/python/grpcio_channelz/channelz_commands.py b/src/python/grpcio_channelz/channelz_commands.py index c42522a6ed0..d467cd70e56 100644 --- a/src/python/grpcio_channelz/channelz_commands.py +++ b/src/python/grpcio_channelz/channelz_commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" import os import shutil diff --git a/src/python/grpcio_health_checking/health_commands.py b/src/python/grpcio_health_checking/health_commands.py index 74df84ad7bf..98fbfb6b035 100644 --- a/src/python/grpcio_health_checking/health_commands.py +++ b/src/python/grpcio_health_checking/health_commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" import os import shutil diff --git a/src/python/grpcio_reflection/reflection_commands.py b/src/python/grpcio_reflection/reflection_commands.py index 07ce59bfca8..48c7c749cea 100644 --- a/src/python/grpcio_reflection/reflection_commands.py +++ b/src/python/grpcio_reflection/reflection_commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" import os import shutil diff --git a/src/python/grpcio_status/status_commands.py b/src/python/grpcio_status/status_commands.py index 25bc4694d9f..c67d5e2cc7d 100644 --- a/src/python/grpcio_status/status_commands.py +++ b/src/python/grpcio_status/status_commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" import os import shutil diff --git a/src/python/grpcio_testing/testing_commands.py b/src/python/grpcio_testing/testing_commands.py index b7374814ac6..18f9ff254b5 100644 --- a/src/python/grpcio_testing/testing_commands.py +++ b/src/python/grpcio_testing/testing_commands.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the GRPC Python setup process.""" +"""Provides setuptools command classes for the GRPC Python setup process.""" import os import shutil diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 9fb3abb4dad..357957afde9 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -11,9 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Provides distutils command classes for the gRPC Python setup process.""" +"""Provides setuptools command classes for the gRPC Python setup process.""" -from distutils import errors as _errors import glob import os import os.path @@ -23,6 +22,7 @@ import shutil import sys import setuptools +from setuptools import errors as _errors from setuptools.command import build_ext from setuptools.command import build_py from setuptools.command import easy_install @@ -305,7 +305,7 @@ class RunInterop(test.test): def finalize_options(self): if self.client and self.server: - raise _errors.DistutilsOptionError( + raise _errors.OptionError( "you may only specify one of client or server" ) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 1dda7149d65..9a6538cca35 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -14,7 +14,6 @@ import collections import contextlib -import distutils.spawn import errno import itertools import os diff --git a/test/distrib/python/test_packages.sh b/test/distrib/python/test_packages.sh index b2984e478a9..989c33a4663 100755 --- a/test/distrib/python/test_packages.sh +++ b/test/distrib/python/test_packages.sh @@ -37,7 +37,7 @@ TESTING_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-testing-[0-9]*.tar VIRTUAL_ENV=$(mktemp -d) python3 -m virtualenv "$VIRTUAL_ENV" PYTHON=$VIRTUAL_ENV/bin/python -"$PYTHON" -m pip install --upgrade six pip wheel +"$PYTHON" -m pip install --upgrade six pip wheel setuptools function validate_wheel_hashes() { for file in "$@"; do diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index c3c5ccc421f..f0b5240f73c 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -141,7 +141,7 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory $ python -m grpc_tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES -To use as a build step in distutils-based projects, you may use the provided +To use as a build step in setuptools-based projects, you may use the provided command class in your :code:`setup.py`: :: @@ -177,5 +177,4 @@ installed). One way to work around this can be found in our Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the command on-setup as desired. -For more information on command classes, consult :code:`distutils` and -:code:`setuptools` documentation. +For more information on command classes, consult :code:`setuptools` documentation. diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py index d5ac317ae5b..2993bd408e9 100644 --- a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py +++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py @@ -17,7 +17,6 @@ build_ext has lots of C/C++ files and normally them one by one. Enabling parallel build helps a lot. """ -import distutils.ccompiler import os try: @@ -66,6 +65,12 @@ def _parallel_compile( def monkeypatch_compile_maybe(): - """Monkeypatching is dumb, but the build speed gain is worth it.""" - if BUILD_EXT_COMPILER_JOBS > 1: + """ + Monkeypatching is dumb, but the build speed gain is worth it. + After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib. + """ + use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "") + if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib": + import distutils.ccompiler # pylint: disable=wrong-import-position + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 9b5f3d828a0..5d6b8a0a974 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from distutils import cygwinccompiler -from distutils import extension -from distutils import util import errno import os import os.path @@ -29,6 +26,7 @@ import sysconfig import pkg_resources import setuptools +from setuptools import Extension from setuptools.command import build_ext # TODO(atash) add flag to disable Cython use @@ -191,12 +189,6 @@ if EXTRA_ENV_LINK_ARGS is None: EXTRA_ENV_LINK_ARGS += " -lpthread" if check_linker_need_libatomic(): EXTRA_ENV_LINK_ARGS += " -latomic" - elif "win32" in sys.platform and sys.version_info < (3, 5): - msvcr = cygwinccompiler.get_msvcr()[0] - EXTRA_ENV_LINK_ARGS += ( - " -static-libgcc -static-libstdc++ -mcrtdll={msvcr}" - " -static -lshlwapi".format(msvcr=msvcr) - ) EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) @@ -228,7 +220,7 @@ if "win32" in sys.platform: elif "linux" in sys.platform or "darwin" in sys.platform: DEFINE_MACROS += (("HAVE_PTHREAD", 1),) -# By default, Python3 distutils enforces compatibility of +# By default, Python3 setuptools(distutils) enforces compatibility of # c plugins (.so files) with the OSX version Python was built with. # We need OSX 10.10, the oldest which supports C++ thread_local. if "darwin" in sys.platform: @@ -241,7 +233,7 @@ if "darwin" in sys.platform: os.environ["_PYTHON_HOST_PLATFORM"] = re.sub( r"macosx-[0-9]+\.[0-9]+-(.+)", r"macosx-10.10-\1", - util.get_platform(), + sysconfig.get_platform(), ) @@ -281,7 +273,7 @@ def extension_modules(): os.path.join("grpc_root", "src", "compiler", "proto_parser_helper.cc"), ] + CC_FILES - plugin_ext = extension.Extension( + plugin_ext = Extension( name="grpc_tools._protoc_compiler", sources=plugin_sources, include_dirs=[ diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index e74b80e27c0..e0c12382830 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -21,7 +21,7 @@ set PATH=C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH% python -m pip install --upgrade six @rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 -python -m pip install --upgrade setuptools==44.1.1 +python -m pip install --upgrade setuptools==59.6.0 python -m pip install --upgrade "cython<3.0.0rc1" python -m pip install -rrequirements.txt --user diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index ce3cff865ee..6f2ca8716e0 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -26,7 +26,7 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel} source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc # Needed for building binary distribution wheels -- bdist_wheel -"${PYTHON}" -m pip install --upgrade wheel +"${PYTHON}" -m pip install --upgrade wheel setuptools if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ] then @@ -182,7 +182,7 @@ fix_faulty_universal2_wheel() { } # This is necessary due to https://github.com/pypa/wheel/issues/406. -# distutils incorrectly generates a universal2 artifact that only contains +# wheel incorrectly generates a universal2 artifact that only contains # x86_64 libraries. if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 625bbe12785..9c22cc182b1 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -77,7 +77,7 @@ function venv_relative_python() { fi } -# Distutils toolchain to use depending on the system. +# Toolchain to use depending on the system. function toolchain() { if [ "$(is_mingw)" ]; then echo 'mingw32' @@ -140,7 +140,6 @@ pip_install() { # Pin setuptools to < 60.0.0 to restore the distutil installation, see: # https://github.com/pypa/setuptools/pull/2896 -export SETUPTOOLS_USE_DISTUTILS=stdlib pip_install --upgrade pip==21.3.1 pip_install --upgrade setuptools==59.6.0