diff --git a/.github/workflows/source_distribution_tests.yml b/.github/workflows/source_distribution_tests.yml new file mode 100644 index 0000000000..ee839c749a --- /dev/null +++ b/.github/workflows/source_distribution_tests.yml @@ -0,0 +1,100 @@ +name: Python Tests + +on: + push: + branches: + - main + - '[0-9]+.x' + pull_request: + branches: + - main + - '[0-9]+.x' + workflow_dispatch: + +jobs: + build_wheels: + name: Build Wheels + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} + env: + DOCKER_IMAGE: gcr.io/protobuf-build/bazel/linux@sha256:9dba7773926bb6dce839de098948055c9e80260d7470a7fefb94b6a4fed363ba + + steps: + - uses: actions/checkout@v2 + - name: Set up Cloud SDK + uses: google-github-actions/auth@v0 + with: + credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} + export_environment_variables: true + - name: Use gcloud CLI + run: gcloud info + - name: Configure Docker + run: gcloud auth configure-docker + - name: Pull Docker Image + run: docker pull $DOCKER_IMAGE + - name: Build Wheels + run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist:source_wheel python/dist:test_wheel + - name: Move Wheels + run: mkdir wheels && find _build/out -name 'protobuf*.whl' -o -name 'protobuf*.tar.gz' -exec mv '{}' wheels ';' + - uses: actions/upload-artifact@v3 + with: + name: python-wheels + path: wheels/ + + test_wheels: + name: Test Wheels + needs: build_wheels + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + include: + # Test on all supported operating systems and using the oldest and + # newest versions of python we support + - { os: ubuntu-18.04, python-version: "3.7", architecture: x64 } + - { os: ubuntu-20.04, python-version: "3.10", architecture: x64 } + - { os: macos-11, python-version: "3.7", architecture: x64 } + - { os: macos-12, python-version: "3.10", architecture: x64 } + - { os: windows-2019, python-version: "3.7", architecture: x86 } + - { os: windows-2019, python-version: "3.10", architecture: x86 } + - { os: windows-2019, python-version: "3.7", architecture: x64 } + - { os: windows-2019, python-version: "3.10", architecture: x64 } + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + steps: + - name: Download Wheels + uses: actions/download-artifact@v3 + with: + name: python-wheels + path: wheels + - name: Extract tar + run: tar -xzvf *.tar.gz + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: ${{ matrix.architecture }} + - name: Setup Python venv + run: | + python -m pip install --upgrade pip + python -m venv env + # Windows uses 'Scripts' instead of 'bin' + source env/bin/activate || source env/Scripts/activate + echo "VIRTUAL ENV:" $VIRTUAL_ENV + - name: Install tzdata + run: pip install tzdata + # Only needed on Windows, Linux ships with tzdata. + if: ${{ contains(matrix.os, 'windows') }} + - name: Install numpy + run: pip install numpy + - name: Install Protobuf Wheels + run: pip install -vvv --no-index --find-links wheels protobuf protobuftests + - name: Test that module is importable + run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)' + if: ${{ !matrix.pure_python }} + - name: Run the unit tests + run: | + TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g') + for test in $TESTS; do + python -m unittest -v $test + done diff --git a/WORKSPACE b/WORKSPACE index e984e692c0..5d353c67b6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -71,6 +71,10 @@ load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") rules_fuzzing_init() +load("@rules_pkg//pkg:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() + load("//bazel:system_python.bzl", "system_python") system_python( name = "system_python", @@ -93,4 +97,4 @@ pip_parse( load("@pip_deps//:requirements.bzl", "install_deps") install_deps() -fuzzing_py_install_deps() \ No newline at end of file +fuzzing_py_install_deps() diff --git a/bazel/workspace_deps.bzl b/bazel/workspace_deps.bzl index c42d5da645..84bfafaa3f 100644 --- a/bazel/workspace_deps.bzl +++ b/bazel/workspace_deps.bzl @@ -23,8 +23,8 @@ def upb_deps(): _github_archive, name = "com_google_protobuf", repo = "https://github.com/protocolbuffers/protobuf", - commit = "016ef2393e163cb8a49186adab8e2981476eec37", - sha256 = "ed91f723ba5db42190ca4a2734d1fdd449ceccf259c1112d3e4da14b322680f5", + commit = "d0169ff9d167ff9109c591cf83655f4c906e712d", + sha256 = "957019f9d116764d7ac248f81a92e49d7ed41b5892e7f15074da5cfba0af46d1", patches = ["@upb//bazel:protobuf.patch"], ) @@ -36,6 +36,15 @@ def upb_deps(): sha256 = "5da960e5e5d92394c809629a03af3c7709d2d3d0ca731dacb3a9fb4bf28f7702", ) + rules_pkg_version = "0.7.0" + + maybe( + http_archive, + name = "rules_pkg", + url = "https://github.com/bazelbuild/rules_pkg/archives/refs/tags/{}.tar.gz".format(rules_pkg_version), + sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", + ) + rules_python_version = "0.14.0" # Latest @ November 20, 2022 maybe( diff --git a/cmake/make_cmakelists.py b/cmake/make_cmakelists.py index 47bf85b738..cfb017633f 100755 --- a/cmake/make_cmakelists.py +++ b/cmake/make_cmakelists.py @@ -241,6 +241,9 @@ class WorkspaceFileFunctions(object): def rules_fuzzing_init(self): pass + def rules_pkg_dependencies(self): + pass + def system_python(self, **kwargs): pass diff --git a/python/BUILD b/python/BUILD index 31bbae3d77..28ea3e8587 100644 --- a/python/BUILD +++ b/python/BUILD @@ -29,6 +29,7 @@ load("//python:py_extension.bzl", "py_extension") load("@bazel_skylib//lib:selects.bzl", "selects") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") +load("@rules_pkg//:mappings.bzl", "pkg_files") load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") licenses(["notice"]) @@ -187,8 +188,8 @@ _message_target_compatible_with = { # } # end:google_only -py_extension( - name = "_message", +filegroup( + name = "message_srcs", srcs = [ "convert.c", "convert.h", @@ -212,6 +213,17 @@ py_extension( "unknown_fields.c", "unknown_fields.h", ], +) + +pkg_files( + name = "message_extension", + srcs = [":message_srcs"], + prefix = "google/protobuf/pyext", +) + +py_extension( + name = "_message", + srcs = [":message_srcs"], copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [ # The Python API requires patterns that are ISO C incompatible, like # casts between function pointers and object pointers. diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel index f27ccfe824..3f8701682a 100644 --- a/python/dist/BUILD.bazel +++ b/python/dist/BUILD.bazel @@ -27,6 +27,8 @@ load("//bazel:py_proto_library.bzl", "py_proto_library") load(":dist.bzl", "py_dist", "py_dist_module") load("@bazel_skylib//lib:selects.bzl", "selects") load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION") +load("@rules_pkg//:mappings.bzl", "pkg_files") +load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@rules_python//python:packaging.bzl", "py_wheel") load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") @@ -43,7 +45,6 @@ py_proto_library( deps = [ "@com_google_protobuf//:any_proto", "@com_google_protobuf//:api_proto", - "@com_google_protobuf//:compiler_plugin_proto", "@com_google_protobuf//:descriptor_proto", "@com_google_protobuf//:duration_proto", "@com_google_protobuf//:empty_proto", @@ -56,6 +57,11 @@ py_proto_library( ], ) +py_proto_library( + name = "well_known_proto_compiler_py_pb2", + deps = ["@com_google_protobuf//:compiler_plugin_proto"], +) + config_setting( name = "linux_aarch64_release", values = {"cpu": "linux-aarch_64"}, @@ -187,6 +193,38 @@ selects.config_setting_group( ], ) +pkg_files( + name = "generated_wkt", + srcs = [ + ":well_known_proto_py_pb2", + ], + prefix = "google/protobuf", +) + +pkg_files( + name = "generated_wkt_compiler", + srcs = [ + ":well_known_proto_compiler_py_pb2", + ], + prefix = "google/protobuf/compiler", +) + +pkg_tar( + name = "source_wheel", + extension = "tar.gz", + srcs = [ + ":generated_wkt", + ":generated_wkt_compiler", + "setup.py", + "//:LICENSE", + "//python:message_extension", + "@com_google_protobuf//python:python_source_files", + ], + package_file_name = "protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION, + package_dir = "protobuf-%s" % PROTOBUF_PYTHON_VERSION, +) + + py_wheel( name = "binary_wheel", abi = select({ @@ -240,6 +278,7 @@ py_wheel( version = PROTOBUF_PYTHON_VERSION, deps = [ ":message_mod", + ":well_known_proto_compiler_py_pb2", ":well_known_proto_py_pb2", "@com_google_protobuf//:python_srcs", ], @@ -278,6 +317,7 @@ py_wheel( version = PROTOBUF_PYTHON_VERSION, deps = [ ":well_known_proto_py_pb2", + ":well_known_proto_compiler_py_pb2", "@com_google_protobuf//:python_srcs", ], ) diff --git a/python/dist/setup.py b/python/dist/setup.py new file mode 100755 index 0000000000..2eb36e8b55 --- /dev/null +++ b/python/dist/setup.py @@ -0,0 +1,84 @@ +#! /usr/bin/env python +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# See README for usage instructions. + +import glob +import os + +# We must use setuptools, not distutils, because we need to use the +# namespace_packages option for the "google" package. +from setuptools import setup, Extension, find_packages + +def GetVersion(): + """Reads and returns the version from google/protobuf/__init__.py. + + Do not import google.protobuf.__init__ directly, because an installed + protobuf library may be loaded instead. + + Returns: + The version. + """ + + with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: + exec(version_file.read(), globals()) # pylint:disable=exec-used + return __version__ # pylint:disable=undefined-variable + +# Keep this list of dependencies in sync with tox.ini. +install_requires = [] + +setup( + name='protobuf', + version=GetVersion(), + description='Protocol Buffers', + download_url='https://github.com/protocolbuffers/protobuf/releases', + long_description="Protocol Buffers are Google's data interchange format", + url='https://developers.google.com/protocol-buffers/', + project_urls={ + 'Source': 'https://github.com/protocolbuffers/protobuf', + }, + maintainer='protobuf@googlegroups.com', + maintainer_email='protobuf@googlegroups.com', + license='BSD-3-Clause', + classifiers=[ + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + ], + namespace_packages=['google'], + packages=find_packages(), + install_requires=install_requires, + ext_modules= [Extension('google.protobuf.pyext._message', glob.glob('google/protobuf/pyext/*.c'))], + python_requires='>=3.7', +)