Add support and partial CI coverage for Python 3.12

There's a test run in test_python.yml that is non-trivial to get working with
Python 3.12 due to some refactoring of our Docker images that would be needed.
But this change updates everything else to add coverage for Python 3.12.

The main changes necessary to get the builds working were to upgrade some Pip
packages via requirements.txt, including in a patch to `rules_fuzzing` that I
plan to upstream soon. I also had to take an explicit dependency on
`setuptools`.

I removed tox.ini, since it was outdated and we have not been actively
maintaining it.

PiperOrigin-RevId: 580548224

.github/workflows/test_python.yml

@@ -17,7 +17,8 @@ jobs:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
         type: [ Pure, C++]
-        version: ["3.8", "3.9", "3.10", "3.11" ]
+        # TODO: b/309627662 - Add coverage for Python 3.12.
+        version: ["3.8", "3.9", "3.10", "3.11"]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -55,7 +56,7 @@ jobs:
       matrix:
         type: [ Pure, C++]
         # TODO Consider expanding this set of versions.
-        version: [ "3.11" ]
+        version: [ "3.12" ]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test

.github/workflows/test_upb.yml

@@ -100,7 +100,6 @@ jobs:
         with:
           cache: pip
           cache-dependency-path: 'python/requirements.txt'
-          python-version: '3.11' # 3.12 doesn't have setuptools
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/bazel@v2
         with:
@@ -171,23 +170,24 @@ jobs:
           # coverage.
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'binary' }
           - { os: macos-11, python-version: "3.8", architecture: x64, type: 'binary' }
-          - { os: ubuntu-latest, python-version: "3.11", architecture: x64, type: 'binary' }
-          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'binary' }
+          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'binary' }
           - { os: ubuntu-latest, python-version: "3.8", architecture: x64, type: 'source' }
           - { os: macos-11, python-version: "3.8", architecture: x64, type: 'source' }
-          - { os: ubuntu-latest, python-version: "3.11", architecture: x64, type: 'source' }
-          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'source' }
+          - { os: ubuntu-latest, python-version: "3.12", architecture: x64, type: 'source' }
+          - { os: macos-12, python-version: "3.12", architecture: x64, type: 'source' }
 
-          # Windows uses the full API up until Python 3.10, so each of these
-          # jobs tests a distinct binary wheel.
+          # Windows uses the full API up until Python 3.10.
           - { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.12", architecture: x86, type: 'binary' }
           - { os: windows-2019, python-version: "3.8", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary' }
           - { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.12", architecture: x64, type: 'binary' }
     runs-on: ${{ matrix.os }}
     if: ${{ github.event_name != 'pull_request_target' }}
     defaults:
@@ -248,7 +248,7 @@ jobs:
     strategy:
       fail-fast: false   # Don't cancel all jobs if one fails.
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.12"]
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'pull_request_target' }}
     steps:

python/build_targets.bzl

@@ -437,7 +437,6 @@ def build_targets(name):
             "README.md",
             "google/__init__.py",
             "setup.cfg",
-            "tox.ini",
         ],
         strip_prefix = "",
         visibility = ["//python/dist:__pkg__"],
@@ -463,7 +462,6 @@ def build_targets(name):
             "python_version_test.py",
             "setup.cfg",
             "setup.py",
-            "tox.ini",
         ],
         strip_prefix = strip_prefix.from_root(""),
         visibility = ["//pkg:__pkg__"],

python/dist/BUILD.bazel

@@ -5,6 +5,7 @@
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
 
+load("@pip_deps//:requirements.bzl", "requirement")
 load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@rules_python//python:packaging.bzl", "py_wheel")
@@ -281,6 +282,7 @@ genrule(
     srcs = [":source_tarball"],
     outs = ["protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION],
     cmd = """
+        export PYTHONPATH=$$PWD/external/pip_deps_setuptools/site-packages
         set -eux
         tar -xzvf $(location :source_tarball)
         cd protobuf/
@@ -292,6 +294,7 @@ genrule(
         "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
+    tools = [requirement("setuptools")],
 )
 
 py_wheel(
@@ -310,6 +313,7 @@ py_wheel(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     distribution = "protobuf",
     extra_distinfo_files = {
@@ -365,6 +369,7 @@ py_wheel(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     distribution = "protobuf",
     extra_distinfo_files = {

python/dist/setup.py

@@ -58,6 +58,7 @@ setup(
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
     ],
     namespace_packages=['google'],
     packages=find_packages(),

python/protobuf_distutils/setup.py

@@ -34,6 +34,7 @@ setup(
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: Software Development :: Code Generators',
     ],
     description=(

python/requirements.txt

@@ -1 +1,2 @@
-numpy<=1.24.4
+numpy<=1.26.1
+setuptools<=68.2.2

python/setup.py

@@ -389,7 +389,6 @@ if __name__ == '__main__':
     ])
     os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
 
-  # Keep this list of dependencies in sync with tox.ini.
   install_requires = []
 
   setup(
@@ -415,6 +414,7 @@ if __name__ == '__main__':
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
           'Programming Language :: Python :: 3.11',
+          'Programming Language :: Python :: 3.12',
       ],
       namespace_packages=['google'],
       packages=find_packages(

python/tox.ini

@@ -1,21 +0,0 @@
-[tox]
-envlist =
-    py{37,38,39,310}-{cpp,python}
-
-[testenv]
-usedevelop=true
-passenv =
-    CC PYTHONPATH KOKORO_BUILD_ID KOKORO_BUILD_NUMBER
-setenv =
-    cpp: LD_LIBRARY_PATH={toxinidir}/../bazel-bin/src/google
-    cpp: DYLD_LIBRARY_PATH={toxinidir}/../bazel-bin/src/google
-    cpp: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
-    python: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
-commands =
-    python setup.py -q build_py
-    python: python setup.py -q build
-    py{37,38,39,310}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
-    python: python setup.py -q test -q
-    cpp: python setup.py -q test -q --cpp_implementation
-    python: python setup.py -q test_conformance
-    cpp: python setup.py -q test_conformance --cpp_implementation

third_party/rules_fuzzing.patch

@@ -22,4 +22,4 @@ index 01482d4..4b36f4f 100644
 -absl-py==0.11.0 --hash=sha256:b3d9eb5119ff6e0a0125f6dabf2f9fae02f8acae7be70576002fac27235611c5
 -six==1.15.0 --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
 +absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3
-+six==1.15.0
++six==1.16.0 --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254