Merge pull request #1052 from protocolbuffers:srcWhl

PiperOrigin-RevId: 501928730
pull/13171/head
Copybara-Service 2 years ago
commit aaf9e43e4f
  1. 2
      .github/workflows/bazel_tests.yml
  2. 47
      .github/workflows/python_tests.yml
  3. 17
      BUILD
  4. 4
      WORKSPACE
  5. 14
      bazel/workspace_deps.bzl
  6. 8
      cmake/make_cmakelists.py
  7. 22
      python/BUILD
  8. 54
      python/dist/BUILD.bazel
  9. 103
      python/dist/setup.py
  10. 20
      upb/util/BUILD

@ -100,4 +100,4 @@ jobs:
- name: Uninstall python
run: which python3 && sudo mv `which python3` /tmp && ! which python3
- name: Run tests
run: cd ${{ github.workspace }} && bazel test --test_output=errors $BAZEL_CACHE $BAZEL_CACHE_AUTH //python/...
run: cd ${{ github.workspace }} && bazel test --test_output=errors $BAZEL_CACHE $BAZEL_CACHE_AUTH //python/... -- -//python/dist:source_wheel

@ -33,9 +33,9 @@ jobs:
- name: Pull Docker Image
run: docker pull $DOCKER_IMAGE
- name: Build Wheels
run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist python/dist:test_wheel
run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist python/dist:test_wheel python/dist:source_wheel
- name: Move Wheels
run: mkdir wheels && find _build/out -name 'protobuf*.whl' -exec mv '{}' wheels ';'
run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf*.tar.gz' \) -exec mv '{}' wheels ';'
- uses: actions/upload-artifact@v3
with:
name: python-wheels
@ -51,20 +51,25 @@ jobs:
# Linux and Mac use the limited API, so all Python versions will use a single wheel.
# As a result we can just test the oldest and newest supported Python versions and assume
# this gives us sufficient test coverage.
- { os: ubuntu-18-large, python-version: "3.7", architecture: x64 }
- { os: macos-11, python-version: "3.7", architecture: x64 }
- { os: ubuntu-20-large, python-version: "3.10", architecture: x64 }
- { os: macos-12, python-version: "3.10", architecture: x64 }
- { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'binary' }
- { os: macos-11, python-version: "3.7", architecture: x64, type: 'binary' }
- { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'binary' }
- { os: macos-12, python-version: "3.10", architecture: x64, type: 'binary' }
- { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'source' }
- { os: macos-11, python-version: "3.7", architecture: x64, type: 'source' }
- { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'source' }
- { os: macos-12, python-version: "3.10", architecture: x64, type: 'source' }
# Windows uses the full API up until Python 3.10, so each of these
# jobs tests a distinct binary wheel.
- { os: windows-2019-large, python-version: "3.7", architecture: x86 }
- { os: windows-2019-large, python-version: "3.8", architecture: x86 }
- { os: windows-2019-large, python-version: "3.9", architecture: x86 }
- { os: windows-2019-large, python-version: "3.10", architecture: x86 }
- { os: windows-2019-large, python-version: "3.7", architecture: x64 }
- { os: windows-2019-large, python-version: "3.8", architecture: x64 }
- { os: windows-2019-large, python-version: "3.9", architecture: x64 }
- { os: windows-2019-large, python-version: "3.10", architecture: x64 }
- { os: windows-2019-large, python-version: "3.7", architecture: x86, type: 'binary' }
- { os: windows-2019-large, python-version: "3.8", architecture: x86, type: 'binary' }
- { os: windows-2019-large, python-version: "3.9", architecture: x86, type: 'binary' }
- { os: windows-2019-large, python-version: "3.10", architecture: x86, type: 'binary' }
- { os: windows-2019-large, python-version: "3.7", architecture: x64, type: 'binary' }
- { os: windows-2019-large, python-version: "3.8", architecture: x64, type: 'binary' }
- { os: windows-2019-large, python-version: "3.9", architecture: x64, type: 'binary' }
- { os: windows-2019-large, python-version: "3.10", architecture: x64, type: 'binary' }
runs-on: ${{ matrix.os }}
defaults:
run:
@ -92,11 +97,21 @@ jobs:
if: ${{ contains(matrix.os, 'windows') }}
- name: Install numpy
run: pip install numpy
- name: Install Protobuf Wheels
run: pip install -vvv --no-index --find-links wheels protobuf protobuftests
- name: Install Protobuf Binary Wheel
run: pip install -vvv --no-index --find-links wheels protobuf
if: ${{ matrix.type == 'binary' }}
- name: Install Protobuf Source Wheel
run: |
cd wheels
tar -xzvf *.tar.gz
cd protobuf-*/
pip install .
if: ${{ matrix.type == 'source' }}
- name: Test that module is importable
run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)'
if: ${{ !matrix.pure_python }}
- name: Install Protobuf Test Wheel
run: pip install -vvv --no-index --find-links wheels protobuftests
- name: Run the unit tests
run: |
TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g')

17
BUILD

@ -54,6 +54,7 @@ load(
"//bazel:amalgamation.bzl",
"upb_amalgamation",
)
load("@rules_pkg//:mappings.bzl", "pkg_files")
# end:github_only
licenses(["notice"])
@ -1188,6 +1189,22 @@ filegroup(
visibility = ["//cmake:__pkg__"],
)
pkg_files(
name = "upb_source_files",
srcs = glob(
[
"upb/**/*.c",
"upb/**/*.h",
"upb/**/*.hpp",
"upb/**/*.inc",
],
exclude = ["upb/conformance_upb.c"],
),
strip_prefix = "",
visibility = ["//python/dist:__pkg__"],
)
# end:github_only
# begin:google_only

@ -71,6 +71,10 @@ load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
rules_fuzzing_init()
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
rules_pkg_dependencies()
load("//bazel:system_python.bzl", "system_python")
system_python(
name = "system_python",

@ -23,8 +23,8 @@ def upb_deps():
_github_archive,
name = "com_google_protobuf",
repo = "https://github.com/protocolbuffers/protobuf",
commit = "016ef2393e163cb8a49186adab8e2981476eec37",
sha256 = "ed91f723ba5db42190ca4a2734d1fdd449ceccf259c1112d3e4da14b322680f5",
sha256 = "bc4add2a4bf70d6db89ddde404264cbf5586ecf99c0bc8d4f1828eb016e4731e",
commit = "761492746a64cd5ea7a53f3028e42452ca74ff71",
patches = ["@upb//bazel:protobuf.patch"],
)
@ -36,6 +36,16 @@ def upb_deps():
sha256 = "5da960e5e5d92394c809629a03af3c7709d2d3d0ca731dacb3a9fb4bf28f7702",
)
maybe(
http_archive,
name = "rules_pkg",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
"https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
],
sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
)
rules_python_version = "0.14.0" # Latest @ November 20, 2022
maybe(

@ -128,6 +128,9 @@ class BuildFileFunctions(object):
def cc_fuzz_test(self, **kwargs):
pass
def pkg_files(self, **kwargs):
pass
def py_library(self, **kwargs):
pass
@ -182,7 +185,7 @@ class BuildFileFunctions(object):
def select(self, arg_dict):
return []
def glob(self, *args):
def glob(self, *args, **kwargs):
return []
def licenses(self, *args):
@ -250,6 +253,9 @@ class WorkspaceFileFunctions(object):
def rules_fuzzing_init(self):
pass
def rules_pkg_dependencies(self):
pass
def system_python(self, **kwargs):
pass

@ -31,6 +31,10 @@ load("@bazel_skylib//lib:selects.bzl", "selects")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS")
# begin:github_only
load("@rules_pkg//:mappings.bzl", "pkg_files")
# end:github_only
licenses(["notice"])
package(
@ -179,6 +183,12 @@ _message_target_compatible_with = {
"@system_python//:unsupported": ["@platforms//:incompatible"],
"//conditions:default": [],
}
pkg_files(
name = "message_extension",
srcs = [":message_srcs"],
prefix = "python/",
)
# end:github_only
# begin:google_only
# _message_target_compatible_with = {
@ -187,8 +197,8 @@ _message_target_compatible_with = {
# }
# end:google_only
py_extension(
name = "_message",
filegroup(
name = "message_srcs",
srcs = [
"convert.c",
"convert.h",
@ -212,6 +222,14 @@ py_extension(
"unknown_fields.c",
"unknown_fields.h",
],
# begin:google_only
# compatible_with = ["//buildenv/target:non_prod"],
# end:google_only
)
py_extension(
name = "_message",
srcs = [":message_srcs"],
copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [
# The Python API requires patterns that are ISO C incompatible, like
# casts between function pointers and object pointers.

@ -27,6 +27,8 @@ load("//bazel:py_proto_library.bzl", "py_proto_library")
load(":dist.bzl", "py_dist", "py_dist_module")
load("@bazel_skylib//lib:selects.bzl", "selects")
load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION")
load("@rules_pkg//:mappings.bzl", "pkg_files")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("@rules_python//python:packaging.bzl", "py_wheel")
load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION")
@ -43,7 +45,6 @@ py_proto_library(
deps = [
"@com_google_protobuf//:any_proto",
"@com_google_protobuf//:api_proto",
"@com_google_protobuf//:compiler_plugin_proto",
"@com_google_protobuf//:descriptor_proto",
"@com_google_protobuf//:duration_proto",
"@com_google_protobuf//:empty_proto",
@ -56,6 +57,11 @@ py_proto_library(
],
)
py_proto_library(
name = "plugin_py_pb2",
deps = ["@com_google_protobuf//:compiler_plugin_proto"],
)
config_setting(
name = "linux_aarch64_release",
values = {"cpu": "linux-aarch_64"},
@ -187,6 +193,50 @@ selects.config_setting_group(
],
)
pkg_files(
name = "generated_wkt",
srcs = [
":well_known_proto_py_pb2",
"//:descriptor_upb_proto",
"//:descriptor_upb_proto_reflection",
],
prefix = "google/protobuf",
)
pkg_files(
name = "generated_wkt_compiler",
srcs = [
":plugin_py_pb2",
],
prefix = "google/protobuf/compiler",
)
pkg_files(
name = "utf8_range_source_files",
srcs = ["@utf8_range//:utf8_range_srcs"],
prefix = "utf8_range",
)
# NOTE: This package currently only works for macos and ubuntu, MSVC users
# should use a binary wheel.
pkg_tar(
name = "source_wheel",
srcs = [
"setup.py",
":generated_wkt",
":generated_wkt_compiler",
":utf8_range_source_files",
"//:LICENSE",
"//:upb_source_files",
"//python:message_extension",
"//upb/util:util_source_files",
"@com_google_protobuf//python:python_source_files",
],
extension = "tar.gz",
package_dir = "protobuf-%s" % PROTOBUF_PYTHON_VERSION,
package_file_name = "protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION,
)
py_wheel(
name = "binary_wheel",
abi = select({
@ -240,6 +290,7 @@ py_wheel(
version = PROTOBUF_PYTHON_VERSION,
deps = [
":message_mod",
":plugin_py_pb2",
":well_known_proto_py_pb2",
"@com_google_protobuf//:python_srcs",
],
@ -277,6 +328,7 @@ py_wheel(
}),
version = PROTOBUF_PYTHON_VERSION,
deps = [
":plugin_py_pb2",
":well_known_proto_py_pb2",
"@com_google_protobuf//:python_srcs",
],

103
python/dist/setup.py vendored

@ -0,0 +1,103 @@
#! /usr/bin/env python
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# See README for usage instructions.
import glob
import os
import sys
import sysconfig
# We must use setuptools, not distutils, because we need to use the
# namespace_packages option for the "google" package.
from setuptools import setup, Extension, find_packages
def GetVersion():
"""Reads and returns the version from google/protobuf/__init__.py.
Do not import google.protobuf.__init__ directly, because an installed
protobuf library may be loaded instead.
Returns:
The version.
"""
with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file:
file_globals = {}
exec(version_file.read(), file_globals) # pylint:disable=exec-used
return file_globals["__version__"]
current_dir = os.path.dirname(os.path.abspath(__file__))
extra_link_args = []
if sys.platform.startswith('win'):
extra_link_args = ['-static']
setup(
name='protobuf',
version=GetVersion(),
description='Protocol Buffers',
download_url='https://github.com/protocolbuffers/protobuf/releases',
long_description="Protocol Buffers are Google's data interchange format",
url='https://developers.google.com/protocol-buffers/',
project_urls={
'Source': 'https://github.com/protocolbuffers/protobuf',
},
maintainer='protobuf@googlegroups.com',
maintainer_email='protobuf@googlegroups.com',
license='BSD-3-Clause',
classifiers=[
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
],
namespace_packages=['google'],
packages=find_packages(),
install_requires=[],
ext_modules=[
Extension(
'google._upb._message',
glob.glob('google/protobuf/*.c')
+ glob.glob('python/*.c')
+ glob.glob('upb/**/*.c')
+ glob.glob('utf8_range/*.c'),
include_dirs=[current_dir, os.path.join(current_dir, 'utf8_range')],
language='c',
extra_link_args=extra_link_args,
)
],
python_requires='>=3.7',
)

@ -4,6 +4,10 @@ load(
"upb_proto_reflection_library",
)
# begin:github_only
load("@rules_pkg//:mappings.bzl", "pkg_files")
# end:github_only
licenses(["notice"])
# Def to Proto
@ -131,3 +135,19 @@ cc_test(
"@com_google_googletest//:gtest_main",
],
)
# begin:github_only
pkg_files(
name = "util_source_files",
srcs = [
"compare.c",
"compare.h",
"def_to_proto.c",
"def_to_proto.h",
"required_fields.c",
"required_fields.h",
],
prefix = "upb/util",
visibility = ["//python/dist:__pkg__"],
)
# end:github_only

Loading…
Cancel
Save