Add bazel target for source distribution

pull/13171/head
Deanna Garcia 2 years ago
parent 3bac8780cb
commit 9880136636
  1. 100
      .github/workflows/source_distribution_tests.yml
  2. 6
      WORKSPACE
  3. 13
      bazel/workspace_deps.bzl
  4. 3
      cmake/make_cmakelists.py
  5. 16
      python/BUILD
  6. 42
      python/dist/BUILD.bazel
  7. 84
      python/dist/setup.py

@ -0,0 +1,100 @@
name: Python Tests
on:
push:
branches:
- main
- '[0-9]+.x'
pull_request:
branches:
- main
- '[0-9]+.x'
workflow_dispatch:
jobs:
build_wheels:
name: Build Wheels
runs-on: ubuntu-latest
if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }}
env:
DOCKER_IMAGE: gcr.io/protobuf-build/bazel/linux@sha256:9dba7773926bb6dce839de098948055c9e80260d7470a7fefb94b6a4fed363ba
steps:
- uses: actions/checkout@v2
- name: Set up Cloud SDK
uses: google-github-actions/auth@v0
with:
credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
export_environment_variables: true
- name: Use gcloud CLI
run: gcloud info
- name: Configure Docker
run: gcloud auth configure-docker
- name: Pull Docker Image
run: docker pull $DOCKER_IMAGE
- name: Build Wheels
run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist:source_wheel python/dist:test_wheel
- name: Move Wheels
run: mkdir wheels && find _build/out -name 'protobuf*.whl' -o -name 'protobuf*.tar.gz' -exec mv '{}' wheels ';'
- uses: actions/upload-artifact@v3
with:
name: python-wheels
path: wheels/
test_wheels:
name: Test Wheels
needs: build_wheels
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
include:
# Test on all supported operating systems and using the oldest and
# newest versions of python we support
- { os: ubuntu-18.04, python-version: "3.7", architecture: x64 }
- { os: ubuntu-20.04, python-version: "3.10", architecture: x64 }
- { os: macos-11, python-version: "3.7", architecture: x64 }
- { os: macos-12, python-version: "3.10", architecture: x64 }
- { os: windows-2019, python-version: "3.7", architecture: x86 }
- { os: windows-2019, python-version: "3.10", architecture: x86 }
- { os: windows-2019, python-version: "3.7", architecture: x64 }
- { os: windows-2019, python-version: "3.10", architecture: x64 }
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
steps:
- name: Download Wheels
uses: actions/download-artifact@v3
with:
name: python-wheels
path: wheels
- name: Extract tar
run: tar -xzvf *.tar.gz
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.architecture }}
- name: Setup Python venv
run: |
python -m pip install --upgrade pip
python -m venv env
# Windows uses 'Scripts' instead of 'bin'
source env/bin/activate || source env/Scripts/activate
echo "VIRTUAL ENV:" $VIRTUAL_ENV
- name: Install tzdata
run: pip install tzdata
# Only needed on Windows, Linux ships with tzdata.
if: ${{ contains(matrix.os, 'windows') }}
- name: Install numpy
run: pip install numpy
- name: Install Protobuf Wheels
run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
- name: Test that module is importable
run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)'
if: ${{ !matrix.pure_python }}
- name: Run the unit tests
run: |
TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')
for test in $TESTS; do
python -m unittest -v $test
done

@ -71,6 +71,10 @@ load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
rules_fuzzing_init()
load("@rules_pkg//pkg:deps.bzl", "rules_pkg_dependencies")
rules_pkg_dependencies()
load("//bazel:system_python.bzl", "system_python")
system_python(
name = "system_python",
@ -93,4 +97,4 @@ pip_parse(
load("@pip_deps//:requirements.bzl", "install_deps")
install_deps()
fuzzing_py_install_deps()
fuzzing_py_install_deps()

@ -23,8 +23,8 @@ def upb_deps():
_github_archive,
name = "com_google_protobuf",
repo = "https://github.com/protocolbuffers/protobuf",
commit = "016ef2393e163cb8a49186adab8e2981476eec37",
sha256 = "ed91f723ba5db42190ca4a2734d1fdd449ceccf259c1112d3e4da14b322680f5",
commit = "d0169ff9d167ff9109c591cf83655f4c906e712d",
sha256 = "957019f9d116764d7ac248f81a92e49d7ed41b5892e7f15074da5cfba0af46d1",
patches = ["@upb//bazel:protobuf.patch"],
)
@ -36,6 +36,15 @@ def upb_deps():
sha256 = "5da960e5e5d92394c809629a03af3c7709d2d3d0ca731dacb3a9fb4bf28f7702",
)
rules_pkg_version = "0.7.0"
maybe(
http_archive,
name = "rules_pkg",
url = "https://github.com/bazelbuild/rules_pkg/archives/refs/tags/{}.tar.gz".format(rules_pkg_version),
sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
)
rules_python_version = "0.14.0" # Latest @ November 20, 2022
maybe(

@ -241,6 +241,9 @@ class WorkspaceFileFunctions(object):
def rules_fuzzing_init(self):
pass
def rules_pkg_dependencies(self):
pass
def system_python(self, **kwargs):
pass

@ -29,6 +29,7 @@ load("//python:py_extension.bzl", "py_extension")
load("@bazel_skylib//lib:selects.bzl", "selects")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
load("@rules_pkg//:mappings.bzl", "pkg_files")
load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS")
licenses(["notice"])
@ -187,8 +188,8 @@ _message_target_compatible_with = {
# }
# end:google_only
py_extension(
name = "_message",
filegroup(
name = "message_srcs",
srcs = [
"convert.c",
"convert.h",
@ -212,6 +213,17 @@ py_extension(
"unknown_fields.c",
"unknown_fields.h",
],
)
pkg_files(
name = "message_extension",
srcs = [":message_srcs"],
prefix = "google/protobuf/pyext",
)
py_extension(
name = "_message",
srcs = [":message_srcs"],
copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [
# The Python API requires patterns that are ISO C incompatible, like
# casts between function pointers and object pointers.

@ -27,6 +27,8 @@ load("//bazel:py_proto_library.bzl", "py_proto_library")
load(":dist.bzl", "py_dist", "py_dist_module")
load("@bazel_skylib//lib:selects.bzl", "selects")
load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION")
load("@rules_pkg//:mappings.bzl", "pkg_files")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("@rules_python//python:packaging.bzl", "py_wheel")
load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION")
@ -43,7 +45,6 @@ py_proto_library(
deps = [
"@com_google_protobuf//:any_proto",
"@com_google_protobuf//:api_proto",
"@com_google_protobuf//:compiler_plugin_proto",
"@com_google_protobuf//:descriptor_proto",
"@com_google_protobuf//:duration_proto",
"@com_google_protobuf//:empty_proto",
@ -56,6 +57,11 @@ py_proto_library(
],
)
py_proto_library(
name = "well_known_proto_compiler_py_pb2",
deps = ["@com_google_protobuf//:compiler_plugin_proto"],
)
config_setting(
name = "linux_aarch64_release",
values = {"cpu": "linux-aarch_64"},
@ -187,6 +193,38 @@ selects.config_setting_group(
],
)
pkg_files(
name = "generated_wkt",
srcs = [
":well_known_proto_py_pb2",
],
prefix = "google/protobuf",
)
pkg_files(
name = "generated_wkt_compiler",
srcs = [
":well_known_proto_compiler_py_pb2",
],
prefix = "google/protobuf/compiler",
)
pkg_tar(
name = "source_wheel",
extension = "tar.gz",
srcs = [
":generated_wkt",
":generated_wkt_compiler",
"setup.py",
"//:LICENSE",
"//python:message_extension",
"@com_google_protobuf//python:python_source_files",
],
package_file_name = "protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION,
package_dir = "protobuf-%s" % PROTOBUF_PYTHON_VERSION,
)
py_wheel(
name = "binary_wheel",
abi = select({
@ -240,6 +278,7 @@ py_wheel(
version = PROTOBUF_PYTHON_VERSION,
deps = [
":message_mod",
":well_known_proto_compiler_py_pb2",
":well_known_proto_py_pb2",
"@com_google_protobuf//:python_srcs",
],
@ -278,6 +317,7 @@ py_wheel(
version = PROTOBUF_PYTHON_VERSION,
deps = [
":well_known_proto_py_pb2",
":well_known_proto_compiler_py_pb2",
"@com_google_protobuf//:python_srcs",
],
)

@ -0,0 +1,84 @@
#! /usr/bin/env python
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# See README for usage instructions.
import glob
import os
# We must use setuptools, not distutils, because we need to use the
# namespace_packages option for the "google" package.
from setuptools import setup, Extension, find_packages
def GetVersion():
"""Reads and returns the version from google/protobuf/__init__.py.
Do not import google.protobuf.__init__ directly, because an installed
protobuf library may be loaded instead.
Returns:
The version.
"""
with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file:
exec(version_file.read(), globals()) # pylint:disable=exec-used
return __version__ # pylint:disable=undefined-variable
# Keep this list of dependencies in sync with tox.ini.
install_requires = []
setup(
name='protobuf',
version=GetVersion(),
description='Protocol Buffers',
download_url='https://github.com/protocolbuffers/protobuf/releases',
long_description="Protocol Buffers are Google's data interchange format",
url='https://developers.google.com/protocol-buffers/',
project_urls={
'Source': 'https://github.com/protocolbuffers/protobuf',
},
maintainer='protobuf@googlegroups.com',
maintainer_email='protobuf@googlegroups.com',
license='BSD-3-Clause',
classifiers=[
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
],
namespace_packages=['google'],
packages=find_packages(),
install_requires=install_requires,
ext_modules= [Extension('google.protobuf.pyext._message', glob.glob('google/protobuf/pyext/*.c'))],
python_requires='>=3.7',
)
Loading…
Cancel
Save