Add CI runs for testing upb Python wheels

This restores the Python wheel CI runs from the old upb repo with only minor
changes. I had to update a path in one of the `py_wheel` rules and also make a
slight tweak to ensure that the `descriptor.upb_minitable.{h,c}` files make it
into the source wheels. The change in text_format_test.py is not strictly
necessary but is a small simplification I made while I was trying to debug an
issue with CRLF newlines.

I had to update test_util.py to use `importlib` to access the golden files from
the installed `protobuftests` package. I suspect the previous incarnation of
thse test runs was somehow reading the goldens from the repo checkout, but I
think the intention is to read them from `protobuftests` instead. This was a
bit tricky to get working because Python versions before 3.9 do not support
`importlib.resources.files()`. I set up the code to fall back on
`importlib.resources.open_binary()` in that case, but that function does not
support subdirectories, so this required putting an `__init__.py` file inside
the `testdata` directory to make sure it is treated as a Python package.

PiperOrigin-RevId: 567366695

.github/workflows/test_upb.yml

@@ -123,7 +123,7 @@ jobs:
   build_wheels:
     name: Build Wheels
     runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'pull_request' }}
+    if: ${{ github.event_name != 'pull_request_target' }}
     steps:
       - name: Checkout pending changes
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -147,3 +147,124 @@ jobs:
           name: requirements
           # Tests shouldn't have access to the whole upb repo, upload the one file we need
           path: upb/python/requirements.txt
+
+  test_wheels:
+    name: Test Wheels
+    needs: build_wheels
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          # Linux and Mac use the limited API, so all Python versions will use
+          # a single wheel. As a result we can just test the oldest and newest
+          # supported Python versions and assume this gives us sufficient test
+          # coverage.
+          - { os: ubuntu-18, python-version: "3.8", architecture: x64, type: 'binary' }
+          - { os: macos-11, python-version: "3.8", architecture: x64, type: 'binary' }
+          - { os: ubuntu-20, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'binary' }
+          - { os: ubuntu-18, python-version: "3.8", architecture: x64, type: 'source' }
+          - { os: macos-11, python-version: "3.8", architecture: x64, type: 'source' }
+          - { os: ubuntu-20, python-version: "3.11", architecture: x64, type: 'source' }
+          - { os: macos-12, python-version: "3.11", architecture: x64, type: 'source' }
+
+          # Windows uses the full API up until Python 3.10, so each of these
+          # jobs tests a distinct binary wheel.
+          - { os: windows-2019, python-version: "3.8", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.9", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.10", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.11", architecture: x86, type: 'binary' }
+          - { os: windows-2019, python-version: "3.8", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.9", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.10", architecture: x64, type: 'binary' }
+          - { os: windows-2019, python-version: "3.11", architecture: x64, type: 'binary' }
+    runs-on: ${{ matrix.os }}
+    if: ${{ github.event_name != 'pull_request_target' }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Download Wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: python-wheels
+          path: wheels
+      - name: Download Requirements
+        uses: actions/download-artifact@v3
+        with:
+          name: requirements
+          path: requirements
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: ${{ matrix.architecture }}
+      - name: Setup Python venv
+        run: |
+          python -m pip install --upgrade pip
+          python -m venv env
+          # Windows uses 'Scripts' instead of 'bin'
+          source env/bin/activate || source env/Scripts/activate
+          echo "VIRTUAL ENV:" $VIRTUAL_ENV
+      - name: Install tzdata
+        run: pip install tzdata
+        # Only needed on Windows, Linux ships with tzdata.
+        if: ${{ contains(matrix.os, 'windows') }}
+      - name: Install requirements
+        run: pip install -r requirements/requirements.txt
+      - name: Install Protobuf Binary Wheel
+        run: pip install -vvv --no-index --find-links wheels protobuf
+        if: ${{ matrix.type == 'binary' }}
+      - name: Install Protobuf Source Wheel
+        run: |
+          cd wheels
+          tar -xzvf *.tar.gz
+          cd protobuf-*/
+          pip install .
+        if: ${{ matrix.type == 'source' }}
+      - name: Test that module is importable
+        run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)'
+      - name: Install Protobuf Test Wheel
+        run: pip install -vvv --no-index --find-links wheels protobuftests
+      - name: Run the unit tests
+        run: |
+          TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')
+          for test in $TESTS; do
+            python -m unittest -v $test
+          done
+
+  test_pure_python_wheels:
+    name: Test Pure Python Wheels
+    needs: build_wheels
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        python-version: ["3.8", "3.11"]
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request_target' }}
+    steps:
+      - name: Download Wheels
+        uses: actions/download-artifact@v3
+        with:
+          name: python-wheels
+          path: wheels
+      - name: Delete Binary Wheels
+        run: find wheels -type f | grep -v none-any | xargs rm
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup Python venv
+        run: |
+          python -m pip install --upgrade pip
+          python -m venv env
+          source env/bin/activate
+          echo "VIRTUAL ENV:" $VIRTUAL_ENV
+      - name: Install numpy
+        run: pip install numpy
+      - name: Install Protobuf Wheels
+        run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
+      - name: Run the unit tests
+        run: |
+          TESTS=$(pip show -f protobuftests | grep _test.py | sed 's,/,.,g' | sed -E 's,.py$,,g')
+          for test in $TESTS; do
+            python -m unittest -v $test
+          done

WORKSPACE

@@ -150,9 +150,6 @@ load("@system_python//:pip.bzl", "pip_parse")
 pip_parse(
     name = "pip_deps",
     requirements = "//upb/python:requirements.txt",
-    requirements_overrides = {
-        "3.11": "//upb/python:requirements_311.txt",
-    },
 )
 
 load("@pip_deps//:requirements.bzl", "install_deps")

python/google/protobuf/internal/test_util.py

@@ -13,10 +13,12 @@ This is intentionally modeled on C++ code in
 
 __author__ = 'robinson@google.com (Will Robinson)'
 
+import importlib.resources
 import numbers
 import operator
 import os.path
 
+from google.protobuf import testdata
 from google.protobuf import unittest_import_pb2
 from google.protobuf import unittest_pb2
 
@@ -607,21 +609,22 @@ def GoldenFile(filename):
       return open(full_path, 'rb')
     path = os.path.join(path, '..')
 
-  # Search internally.
-  path = '.'
-  full_path = os.path.join(path, 'third_party/py/google/protobuf/testdata',
-                           filename)
-  if os.path.exists(full_path):
-    # Found it.  Load the golden file from the testdata directory.
-    return open(full_path, 'rb')
-
   # Search for cross-repo path.
-  full_path = os.path.join('external/com_google_protobuf/src/google/protobuf/testdata',
-                           filename)
+  full_path = os.path.join(
+      'external/com_google_protobuf/src/google/protobuf/testdata', filename
+  )
   if os.path.exists(full_path):
     # Found it.  Load the golden file from the testdata directory.
     return open(full_path, 'rb')
 
+  try:
+    full_path = importlib.resources.files(testdata) / filename
+    if os.path.exists(full_path):
+      return open(full_path, 'rb')
+  except AttributeError:
+    # Fallback for Python < 3.9
+    return importlib.resources.open_binary(testdata, filename)
+
   raise RuntimeError(
       'Could not find golden files.  This test must be run from within the '
       'protobuf source package so that it can read test data files from the '

python/google/protobuf/internal/text_format_test.py

@@ -53,8 +53,7 @@ class TextFormatBase(unittest.TestCase):
 
   def ReadGolden(self, golden_filename):
     with test_util.GoldenFile(golden_filename) as f:
-      return (f.readlines() if str is bytes else  # PY3
-              [golden_line.decode('utf-8') for golden_line in f])
+      return [golden_line.decode('utf-8') for golden_line in f]
 
   def CompareToGoldenFile(self, text, golden_filename):
     golden_lines = self.ReadGolden(golden_filename)

python/setup.py

@@ -426,7 +426,10 @@ if __name__ == '__main__':
           'Programming Language :: Python',
           'Programming Language :: Python :: 3',
           'Programming Language :: Python :: 3.7',
+          # LINT.IfChange
+          # Remove importlib fallback path when we drop Python 3.8 support.
           'Programming Language :: Python :: 3.8',
+          # LINT.ThenChange(//depot/google3/google/protobuf/internal/test_util.py)
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
       ],
@@ -435,7 +438,8 @@ if __name__ == '__main__':
           exclude=[
               'import_test_package',
               'protobuf_distutils',
-          ],),
+          ],
+      ),
       test_suite='google.protobuf.internal',
       cmdclass={
           'clean': CleanCmd,

upb/python/dist/BUILD.bazel

@@ -199,6 +199,7 @@ pkg_files(
     name = "generated_wkt",
     srcs = [
         ":well_known_proto_py_pb2",
+        "//upb:descriptor_upb_minitable_proto",
         "//upb:descriptor_upb_proto",
         "//upb:descriptor_upb_proto_reflection",
     ],
@@ -336,7 +337,7 @@ py_wheel(
         "//conditions:default": "cp" + SYSTEM_PYTHON_VERSION,
     }),
     strip_path_prefixes = [
-        "python/dist/",
+        "upb/python/dist/",
         "python/",
         "src/",
     ],

upb/python/requirements.txt

@@ -1 +1 @@
-numpy==1.21.6
+numpy<=1.24.4

upb/python/requirements_311.txt

@@ -1 +0,0 @@
-numpy==1.23.5