From ffdb58dd5f938eb6fec3073c9fb7ec81870d76d6 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Fri, 22 Sep 2023 09:57:29 -0700 Subject: [PATCH] [Python 3.12] Support Python 3.12 (#34398) Support Python 3.12. ### Testing * Passed all Distribution Tests. * Also tested locally by installing 3.12 artifact. --- setup.py | 1 + src/python/grpcio/_spawn_patch.py | 6 ++-- src/python/grpcio/support.py | 4 +-- src/python/grpcio_channelz/setup.py | 1 + src/python/grpcio_health_checking/setup.py | 1 + src/python/grpcio_reflection/setup.py | 1 + src/python/grpcio_status/setup.py | 1 + .../dockerimage_current_versions.bzl | 12 +++---- tools/dockerfile/README.md | 6 ++++ ...rtifact_python_linux_armv7.current_version | 2 +- .../Dockerfile | 2 ++ ...thon_manylinux2014_aarch64.current_version | 2 +- .../Dockerfile | 4 ++- ...t_python_manylinux2014_x64.current_version | 2 +- .../Dockerfile | 4 ++- ...t_python_manylinux2014_x86.current_version | 2 +- .../Dockerfile | 4 ++- ...t_python_musllinux_1_1_x64.current_version | 2 +- .../Dockerfile | 4 ++- ...t_python_musllinux_1_1_x86.current_version | 2 +- .../Dockerfile | 4 ++- tools/dockerfile/push_testing_images.sh | 2 +- .../install_python_interpreters.ps1 | 17 ++++++++++ .../helper_scripts/prepare_build_macos_rc | 8 +++++ tools/run_tests/artifacts/artifact_targets.py | 31 +++++++++++++------ .../artifacts/build_artifact_python.bat | 12 ++++++- .../artifacts/build_artifact_python.sh | 2 +- 27 files changed, 104 insertions(+), 35 deletions(-) diff --git a/setup.py b/setup.py index f42f459df87..94aa7793a79 100644 --- a/setup.py +++ b/setup.py @@ -109,6 +109,7 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", ] diff --git a/src/python/grpcio/_spawn_patch.py b/src/python/grpcio/_spawn_patch.py index e5e1f1ef85c..cb1f90972f4 100644 --- a/src/python/grpcio/_spawn_patch.py +++ b/src/python/grpcio/_spawn_patch.py @@ -30,7 +30,7 @@ MAX_COMMAND_LENGTH = 8191 _classic_spawn = ccompiler.CCompiler.spawn -def _commandfile_spawn(self, command): +def _commandfile_spawn(self, command, **kwargs): command_length = sum([len(arg) for arg in command]) if os.name == "nt" and command_length > MAX_COMMAND_LENGTH: # Even if this command doesn't support the @command_file, it will @@ -51,11 +51,11 @@ def _commandfile_spawn(self, command): command_file.write(" \n".join(escaped_args)) modified_command = command[:1] + ["@{}".format(command_filename)] try: - _classic_spawn(self, modified_command) + _classic_spawn(self, modified_command, **kwargs) finally: shutil.rmtree(temporary_directory) else: - _classic_spawn(self, command) + _classic_spawn(self, command, **kwargs) def monkeypatch_spawn(): diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py index cab3be17e87..c30c38f6bf3 100644 --- a/src/python/grpcio/support.py +++ b/src/python/grpcio/support.py @@ -117,11 +117,11 @@ def diagnose_build_ext_error(build_ext, error, formatted): diagnostic = _ERROR_DIAGNOSES.get(type(error)) if diagnostic is None: raise commands.CommandError( - "\n\nWe could not diagnose your build failure. If you are unable to" + "\n\nWe could not diagnose your build failure with type {}. If you are unable to" " proceed, please file an issue at http://www.github.com/grpc/grpc" " with `[Python install]` in the title; please attach the whole log" " (including everything that may have appeared above the Python" - " backtrace).\n\n{}".format(formatted) + " backtrace).\n\n{}".format(type(error), formatted) ) else: diagnostic(build_ext, error) diff --git a/src/python/grpcio_channelz/setup.py b/src/python/grpcio_channelz/setup.py index 7a7ce0dfba2..1da8b043406 100644 --- a/src/python/grpcio_channelz/setup.py +++ b/src/python/grpcio_channelz/setup.py @@ -56,6 +56,7 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", ] diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 446b2a10e15..7fd0fec4201 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -55,6 +55,7 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", ] diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index a92e11f807d..d3da0bc7426 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -56,6 +56,7 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", ] diff --git a/src/python/grpcio_status/setup.py b/src/python/grpcio_status/setup.py index 1fce94d2773..ad647e5db8b 100644 --- a/src/python/grpcio_status/setup.py +++ b/src/python/grpcio_status/setup.py @@ -55,6 +55,7 @@ CLASSIFIERS = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: Apache Software License", ] diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 8ab20a882a2..a4c37412693 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -65,12 +65,12 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/grpc_artifact_centos6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x64@sha256:3285047265ea2b7c5d4df4c769b2d05f56288d947c75e16d27ae2dee693f791b", "tools/dockerfile/grpc_artifact_centos6_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x86@sha256:19783239da92208f0f39cf563529cd02e889920497ef81c60d20391fa998af62", "tools/dockerfile/grpc_artifact_protoc_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_protoc_aarch64@sha256:1a3957f32e81259e6f3c602bd67feb132ebc5a5f23e9cb0bf63ba34b91185982", - "tools/dockerfile/grpc_artifact_python_linux_armv7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7@sha256:4f817dece74bbdc7c4fccdc9b0a25cefb9101781a60bf0bb827e533e79f9b1f2", - "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:d56ea4394ea5ea9d09f940d1dba31e6196a8e919f60c6a4966a9192faa997f11", - "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:67ab746e6da576606ebf41ad81027ad897544445fb93d5d5ca5f9d9b5428ec84", - "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:993a963ac3985f8634951e1573d34e24b3868dfff3ad4ae4875dd2c47b73224f", - "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:09bf18cc793d55cfc48d8e88b8b6e6914e9df2b35ec417fe77a4e20bfa251df7", - "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:0512449e7d218c7687eb447701c8c6a33153a722722b76b2423ec58440a027de", + "tools/dockerfile/grpc_artifact_python_linux_armv7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7@sha256:f109d6c22cadb053f6843a66ee827d74f34d6cbf75a32f455a9da099ed1bdc9c", + "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:ef3ae7991763a499b7a0317fd628d1c9d1eef7931fb160361d230f2009febdc3", + "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:0725226dfb1f707ed3cb2d7cafb2376c33fa89d58577bf80e1d76a0c3bc92c64", + "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:8d00dc61594f4fa05ada5c38d55627f139a73cd3a6e4fbc0907a866f2a5f5b17", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:a53a196823862af50ec05d0c5092f765438b488772d1bf53f431cbc98233403d", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:3b0892b4d7c751781fe7947d785837cb5a9c9768519066a4e755838acdfd3ad6", "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878", "tools/dockerfile/interoptest/grpc_interop_cxx.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_cxx@sha256:5a0b1690e6eb63d0904786da9daf6e9a66c901dc7b73d97b4652c05473401790", "tools/dockerfile/interoptest/grpc_interop_dart.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_dart@sha256:5e335005b27709f0882c5723affafa55094bd27a0cda7ce91c718deed157f2bb", diff --git a/tools/dockerfile/README.md b/tools/dockerfile/README.md index 1ccaa221e1d..89c599d6079 100644 --- a/tools/dockerfile/README.md +++ b/tools/dockerfile/README.md @@ -46,6 +46,12 @@ Rebuild the docker images that have been modified locally and upload the docker artifact registry (note that this won't overwrite the "old" versions of the docker image that are already in artifact registry) ``` +# Install qemu, binformat, and configure binfmt interpreters +sudo apt-get install binfmt-support qemu-user-static + +# Enable different multi-architecture containers by QEMU with Docker +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + tools/dockerfile/push_testing_images.sh ``` diff --git a/tools/dockerfile/grpc_artifact_python_linux_armv7.current_version b/tools/dockerfile/grpc_artifact_python_linux_armv7.current_version index 89842795da9..2ddbcc9ba32 100644 --- a/tools/dockerfile/grpc_artifact_python_linux_armv7.current_version +++ b/tools/dockerfile/grpc_artifact_python_linux_armv7.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7:e083630fb0d91e534b69f2f64dbd47f79ce977e2@sha256:4f817dece74bbdc7c4fccdc9b0a25cefb9101781a60bf0bb827e533e79f9b1f2 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7:0b87638ab97b433c6818be5124dcd89d479b3a8b@sha256:f109d6c22cadb053f6843a66ee827d74f34d6cbf75a32f455a9da099ed1bdc9c \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile index cb1674ef4a9..40a22e6fe54 100644 --- a/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_linux_armv7/Dockerfile @@ -28,6 +28,7 @@ RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.8.8" "3.8.8" /opt/p RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.9.2" "3.9.2" /opt/python/cp39-cp39 RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.10.0" "3.10.0rc1" /opt/python/cp310-cp310 RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.11.0" "3.11.0rc1" /opt/python/cp311-cp311 +RUN /scripts/install_python_for_wheel_crosscompilation.sh "3.12.0" "3.12.0rc2" /opt/python/cp312-cp312 ENV AUDITWHEEL_ARCH armv7l ENV AUDITWHEEL_PLAT linux_armv7l @@ -62,3 +63,4 @@ ENV CXX ${CROSS_TRIPLE}-g++ RUN echo "# file contents removed to avoid resetting PATH set by the docker image" >/etc/profile RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version index 59a73a20c4e..4e434e7cb85 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64:6e2cf9602baccda05d82402885660e028cc3ebc6@sha256:d56ea4394ea5ea9d09f940d1dba31e6196a8e919f60c6a4966a9192faa997f11 +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64:cfc9fe0cb36bb4ec8425982fd7f07e4e14bf3941@sha256:ef3ae7991763a499b7a0317fd628d1c9d1eef7931fb160361d230f2009febdc3 \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile index db73c543fc3..8e1c0751a8a 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM dockcross/manylinux2014-aarch64:20230202-a61e35f +FROM dockcross/manylinux2014-aarch64:20230905-7b2d74f # manylinux_2_17 is the preferred alias of manylinux2014 ENV AUDITWHEEL_PLAT manylinux_2_17_$AUDITWHEEL_ARCH @@ -25,6 +25,7 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<3.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<3.0.0rc1' #================= # Install ccache @@ -42,3 +43,4 @@ RUN unset CMAKE_TOOLCHAIN_FILE && unset AS AR CC CPP CXX LD STRIP OBJCOPY \ && rm -rf ccache-4.5.1 ccache.tar.gz RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version b/tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version index 80f2efc75f2..380db946b27 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64:0af99bbd5b3f24218f6009df916a19fd7cb4f11c@sha256:67ab746e6da576606ebf41ad81027ad897544445fb93d5d5ca5f9d9b5428ec84 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64:e81af0c6d454929785e8126fc993cbd93da5c014@sha256:0725226dfb1f707ed3cb2d7cafb2376c33fa89d58577bf80e1d76a0c3bc92c64 \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile index e9f8c534c15..ae959de2db4 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile @@ -14,7 +14,7 @@ # Docker file for building gRPC manylinux Python artifacts. -FROM quay.io/pypa/manylinux2014_x86_64:2022-08-29-0fd77fa +FROM quay.io/pypa/manylinux2014_x86_64:2023-09-17-ae90a16 # manylinux_2_17 is the preferred alias of manylinux2014 ENV AUDITWHEEL_PLAT manylinux_2_17_$AUDITWHEEL_ARCH @@ -30,6 +30,7 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<3.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<3.0.0rc1' #================= # Install ccache @@ -46,3 +47,4 @@ RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/downloa && rm -rf ccache-4.5.1 ccache.tar.gz RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version b/tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version index 60ded4cf1e3..dfad77a959c 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86:904b062b982ad12aad91173e2fc7809fd44c3924@sha256:993a963ac3985f8634951e1573d34e24b3868dfff3ad4ae4875dd2c47b73224f \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86:fda00560d600eed356f8d6bef2cc8416d0a6b790@sha256:8d00dc61594f4fa05ada5c38d55627f139a73cd3a6e4fbc0907a866f2a5f5b17 \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile index c5435575fca..7736403f098 100644 --- a/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile @@ -14,7 +14,7 @@ # Docker file for building gRPC manylinux Python artifacts. -FROM quay.io/pypa/manylinux2014_i686:2022-08-29-0fd77fa +FROM quay.io/pypa/manylinux2014_i686:2023-09-17-ae90a16 # manylinux_2_17 is the preferred alias of manylinux2014 ENV AUDITWHEEL_PLAT manylinux_2_17_$AUDITWHEEL_ARCH @@ -30,6 +30,7 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<3.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<3.0.0rc1' #================= # Install ccache @@ -47,3 +48,4 @@ RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/downloa RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version index 0393c743da0..277b6bbf89a 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64:0807f8211c3f381efa8684232c087c98a4ca7cb8@sha256:09bf18cc793d55cfc48d8e88b8b6e6914e9df2b35ec417fe77a4e20bfa251df7 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64:2f56fcee2e4964359dddcd57bb9b066ceb8331ec@sha256:a53a196823862af50ec05d0c5092f765438b488772d1bf53f431cbc98233403d \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile index b19d6ca28a5..a6eff9bbe71 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM quay.io/pypa/musllinux_1_1_x86_64:2022-08-29-0fd77fa +FROM quay.io/pypa/musllinux_1_1_x86_64:2023-09-17-ae90a16 ################################### # Install Python build requirements @@ -22,6 +22,7 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<3.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<3.0.0rc1' #================= # Install ccache @@ -38,3 +39,4 @@ RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/downloa && rm -rf ccache-4.5.1 ccache.tar.gz RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version index 4c64f34c70a..4ef8f8d0a36 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86:3bc3eacb8eb4ebce5d13a2d4ecabba9074511d48@sha256:0512449e7d218c7687eb447701c8c6a33153a722722b76b2423ec58440a027de \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86:9308dccacd76fb1782a43d97e6f5062b5b58576d@sha256:3b0892b4d7c751781fe7947d785837cb5a9c9768519066a4e755838acdfd3ad6 \ No newline at end of file diff --git a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86/Dockerfile index 0384dfa1fe0..c7ac549e98f 100644 --- a/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM quay.io/pypa/musllinux_1_1_i686:2022-08-29-0fd77fa +FROM quay.io/pypa/musllinux_1_1_i686:2023-09-17-ae90a16 ################################### # Install Python build requirements @@ -22,6 +22,7 @@ RUN /opt/python/cp38-cp38/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp39-cp39/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp310-cp310/bin/pip install --upgrade 'cython<3.0.0rc1' RUN /opt/python/cp311-cp311/bin/pip install --upgrade 'cython<3.0.0rc1' +RUN /opt/python/cp312-cp312/bin/pip install --upgrade 'cython<3.0.0rc1' #================= # Install ccache @@ -38,3 +39,4 @@ RUN curl -sSL -o ccache.tar.gz https://github.com/ccache/ccache/releases/downloa && rm -rf ccache-4.5.1 ccache.tar.gz RUN git config --global --add safe.directory /var/local/jenkins/grpc +RUN git config --global protocol.file.allow always \ No newline at end of file diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh index 8c136b46ba0..07362bc5bc6 100755 --- a/tools/dockerfile/push_testing_images.sh +++ b/tools/dockerfile/push_testing_images.sh @@ -18,7 +18,7 @@ # they simply provide an easily reproducible environment for running gRPC # tests. -set -e +set -ex cd $(dirname $0)/../.. git_root=$(pwd) diff --git a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 index 6a54c59ce3f..902fa3b0965 100644 --- a/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 +++ b/tools/internal_ci/helper_scripts/install_python_interpreters.ps1 @@ -132,3 +132,20 @@ $Python311x64Config = @{ PythonInstallerHash = "5943d8702e40a5ccd62e5a8d4c8852aa" } Install-Python @Python311x64Config + +# Python 3.12 +$Python312x86Config = @{ + PythonVersion = "3.12.0" + PythonInstaller = "python-3.12.0rc2" + PythonInstallPath = "C:\Python312_32bit" + PythonInstallerHash = "c7bfbde3b07a7bd013043640ca475b8c" +} +Install-Python @Python312x86Config + +$Python312x64Config = @{ + PythonVersion = "3.12.0" + PythonInstaller = "python-3.12.0rc2-amd64" + PythonInstallPath = "C:\Python312" + PythonInstallerHash = "ea1993b5227fa4c8f45a06f5fbdd23b3" +} +Install-Python @Python312x64Config diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 068b41e1983..a0d9c285242 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -168,6 +168,14 @@ then shasum -c /tmp/python_installer_checksum.sha256 time sudo installer -pkg ./python-3.11.0rc1-macos11.pkg -target / fi + + # Install Python 3.12 if it doesn't exist + if [ ! -f "/usr/local/bin/python3.12" ]; then + time curl -O https://www.python.org/ftp/python/3.12.0/python-3.12.0rc2-macos11.pkg + echo "e5dc5511b604b501b892991d257d117bb5757437 python-3.12.0rc2-macos11.pkg" > /tmp/python_installer_checksum.sha256 + shasum -c /tmp/python_installer_checksum.sha256 + time sudo installer -pkg ./python-3.12.0rc2-macos11.pkg -target / + fi fi if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ] diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 657577c47d2..4e4671c2591 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -402,8 +402,9 @@ def targets(): PythonArtifact("manylinux2014", "x64", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x64", "cp39-cp39"), PythonArtifact("manylinux2014", "x64", "cp310-cp310"), + PythonArtifact("manylinux2014", "x64", "cp311-cp311"), PythonArtifact( - "manylinux2014", "x64", "cp311-cp311", presubmit=True + "manylinux2014", "x64", "cp312-cp312", presubmit=True ), PythonArtifact( "manylinux2014", "x86", "cp37-cp37m", presubmit=True @@ -411,8 +412,9 @@ def targets(): PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), PythonArtifact("manylinux2014", "x86", "cp39-cp39"), PythonArtifact("manylinux2014", "x86", "cp310-cp310"), + PythonArtifact("manylinux2014", "x86", "cp311-cp311"), PythonArtifact( - "manylinux2014", "x86", "cp311-cp311", presubmit=True + "manylinux2014", "x86", "cp312-cp312", presubmit=True ), PythonArtifact( "manylinux2014", "aarch64", "cp37-cp37m", presubmit=True @@ -423,18 +425,23 @@ def targets(): PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), + PythonArtifact( + "manylinux2014", "aarch64", "cp312-cp312", presubmit=True + ), PythonArtifact( "linux_extra", "armv7", "cp37-cp37m", presubmit=True ), PythonArtifact("linux_extra", "armv7", "cp38-cp38"), PythonArtifact("linux_extra", "armv7", "cp39-cp39"), PythonArtifact("linux_extra", "armv7", "cp310-cp310"), + PythonArtifact("linux_extra", "armv7", "cp311-cp311"), PythonArtifact( - "linux_extra", "armv7", "cp311-cp311", presubmit=True + "linux_extra", "armv7", "cp312-cp312", presubmit=True ), PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), + PythonArtifact("musllinux_1_1", "x64", "cp311-cp311"), PythonArtifact( - "musllinux_1_1", "x64", "cp311-cp311", presubmit=True + "musllinux_1_1", "x64", "cp312-cp312", presubmit=True ), PythonArtifact( "musllinux_1_1", "x64", "cp37-cp37m", presubmit=True @@ -442,8 +449,9 @@ def targets(): PythonArtifact("musllinux_1_1", "x64", "cp38-cp38"), PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), + PythonArtifact("musllinux_1_1", "x86", "cp311-cp311"), PythonArtifact( - "musllinux_1_1", "x86", "cp311-cp311", presubmit=True + "musllinux_1_1", "x86", "cp312-cp312", presubmit=True ), PythonArtifact( "musllinux_1_1", "x86", "cp37-cp37m", presubmit=True @@ -453,18 +461,21 @@ def targets(): PythonArtifact("macos", "x64", "python3.7", presubmit=True), PythonArtifact("macos", "x64", "python3.8"), PythonArtifact("macos", "x64", "python3.9"), - PythonArtifact("macos", "x64", "python3.10", presubmit=True), + PythonArtifact("macos", "x64", "python3.10"), PythonArtifact("macos", "x64", "python3.11", presubmit=True), + PythonArtifact("macos", "x64", "python3.12", presubmit=True), PythonArtifact("windows", "x86", "Python37_32bit", presubmit=True), - PythonArtifact("windows", "x86", "Python38_32bit"), + PythonArtifact("windows", "x86", "Python38_32bit", presubmit=True), PythonArtifact("windows", "x86", "Python39_32bit"), PythonArtifact("windows", "x86", "Python310_32bit"), - PythonArtifact("windows", "x86", "Python311_32bit", presubmit=True), + PythonArtifact("windows", "x86", "Python311_32bit"), + PythonArtifact("windows", "x86", "Python312_32bit", presubmit=True), PythonArtifact("windows", "x64", "Python37", presubmit=True), - PythonArtifact("windows", "x64", "Python38"), + PythonArtifact("windows", "x64", "Python38", presubmit=True), PythonArtifact("windows", "x64", "Python39"), PythonArtifact("windows", "x64", "Python310"), - PythonArtifact("windows", "x64", "Python311", presubmit=True), + PythonArtifact("windows", "x64", "Python311"), + PythonArtifact("windows", "x64", "Python312", presubmit=True), RubyArtifact("linux", "x86-mingw32", presubmit=True), RubyArtifact("linux", "x64-mingw32", presubmit=True), RubyArtifact("linux", "x64-mingw-ucrt", presubmit=True), diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index e0c12382830..0bfc06d388b 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -19,9 +19,19 @@ set PATH=C:\%1;C:\%1\scripts;%PATH% set PATH=C:\msys64\mingw%2\bin;C:\tools\msys64\mingw%2\bin;%PATH% :end_mingw64_installation +@rem check if building for Python3.7 in windows +set IS_PYTHON_37=false +If "%1" == "Python37_32bit" set IS_PYTHON_37=true +If "%1" == "Python37" set IS_PYTHON_37=true + +@rem Python3.7 in windows doesn't support high version of setuptools +if "%IS_PYTHON_37%" == "true" ( python -m pip install --upgrade six -@rem some artifacts are broken for setuptools 38.5.0. See https://github.com/grpc/grpc/issues/14317 python -m pip install --upgrade setuptools==59.6.0 +) else ( +python -m pip install --upgrade pip six setuptools wheel +) + python -m pip install --upgrade "cython<3.0.0rc1" python -m pip install -rrequirements.txt --user diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index a462ab5493e..d5530dfdead 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -26,7 +26,7 @@ export AUDITWHEEL=${AUDITWHEEL:-auditwheel} source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc # Needed for building binary distribution wheels -- bdist_wheel -"${PYTHON}" -m pip install --upgrade wheel setuptools +"${PYTHON}" -m pip install --upgrade pip wheel setuptools if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ] then