Protobuf Team
3 years ago
committed by
Copybara-Service
parent
6a2c01a7f2
commit
ee6b1abb35
18 changed files with 697 additions and 149 deletions
@ -1,38 +0,0 @@ |
||||
load( |
||||
"//bazel:build_defs.bzl", |
||||
"UPB_DEFAULT_COPTS", |
||||
) |
||||
|
||||
def py_extension(name, srcs, deps = []): |
||||
version_script = name + "_version_script.lds" |
||||
symbol = "PyInit_" + name |
||||
native.genrule( |
||||
name = "gen_" + version_script, |
||||
outs = [version_script], |
||||
cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@", |
||||
) |
||||
|
||||
native.cc_binary( |
||||
name = name, |
||||
srcs = srcs, |
||||
copts = UPB_DEFAULT_COPTS + [ |
||||
# The Python API requires patterns that are ISO C incompatible, like |
||||
# casts between function pointers and object pointers. |
||||
"-Wno-pedantic", |
||||
], |
||||
# We use a linker script to hide all symbols except the entry point for |
||||
# the module. |
||||
linkopts = select({ |
||||
"@platforms//os:linux": ["-Wl,--version-script,$(location :" + version_script + ")"], |
||||
"@platforms//os:macos": [ |
||||
"-Wl,-exported_symbol", |
||||
"-Wl,_" + symbol, |
||||
], |
||||
}), |
||||
linkshared = True, |
||||
linkstatic = True, |
||||
deps = deps + [ |
||||
":" + version_script, |
||||
"@system_python//:python_headers", |
||||
], |
||||
) |
||||
@ -0,0 +1,84 @@ |
||||
"""Helper methods to download different python versions""" |
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") |
||||
|
||||
limited_api_build_file = """ |
||||
cc_library( |
||||
name = "python_headers", |
||||
hdrs = glob(["**/Include/**/*.h"]), |
||||
strip_include_prefix = "Python-{}/Include", |
||||
visibility = ["//visibility:public"], |
||||
) |
||||
""" |
||||
|
||||
def python_source_archive(name, sha256): |
||||
"""Helper method to create a python_headers target that will work for linux and macos. |
||||
|
||||
Args: |
||||
name: The name of the target, should be in the form python_{VERSION} |
||||
sha256: The sha256 of the python package for the specified version |
||||
""" |
||||
version = name.split("-")[1] |
||||
http_archive( |
||||
name = name, |
||||
urls = [ |
||||
"https://www.python.org/ftp/python/{0}/Python-{0}.tgz" |
||||
.format(version), |
||||
], |
||||
sha256 = sha256, |
||||
build_file_content = limited_api_build_file.format(version), |
||||
patch_cmds = [ |
||||
"echo '#define SIZEOF_WCHAR_T 4' > Python-{}/Include/pyconfig.h" |
||||
.format(version), |
||||
], |
||||
) |
||||
|
||||
nuget_build_file = """ |
||||
cc_import( |
||||
name = "python_full_api", |
||||
hdrs = glob(["**/*.h"]), |
||||
shared_library = "python{0}.dll", |
||||
interface_library = "libs/python{0}.lib", |
||||
visibility = ["@upb//python:__pkg__"], |
||||
) |
||||
|
||||
cc_import( |
||||
name = "python_limited_api", |
||||
hdrs = glob(["**/*.h"]), |
||||
shared_library = "python{1}.dll", |
||||
interface_library = "libs/python{1}.lib", |
||||
visibility = ["@upb//python:__pkg__"], |
||||
) |
||||
""" |
||||
|
||||
def python_nuget_package(name, sha256): |
||||
"""Helper method to create full and limited api dependencies for windows using nuget |
||||
|
||||
Args: |
||||
name: The name of the target, should be in the form nuget_python_{CPU}_{VERSION} |
||||
sha256: The sha256 of the nuget package for that version |
||||
""" |
||||
cpu = name.split("_")[2] |
||||
version = name.split("_")[3] |
||||
|
||||
full_api_lib_number = version.split(".")[0] + version.split(".")[1] |
||||
limited_api_lib_number = version.split(".")[0] |
||||
|
||||
folder_name_dict = { |
||||
"i686": "pythonx86", |
||||
"x86-64": "python", |
||||
} |
||||
|
||||
http_archive( |
||||
name = name, |
||||
urls = [ |
||||
"https://www.nuget.org/api/v2/package/{}/{}" |
||||
.format(folder_name_dict[cpu], version), |
||||
], |
||||
sha256 = sha256, |
||||
strip_prefix = "tools", |
||||
build_file_content = |
||||
nuget_build_file.format(full_api_lib_number, limited_api_lib_number), |
||||
type = "zip", |
||||
patch_cmds = ["cp -r include/* ."], |
||||
) |
||||
@ -0,0 +1,152 @@ |
||||
# Copyright (c) 2009-2022, Google LLC |
||||
# All rights reserved. |
||||
# |
||||
# Redistribution and use in source and binary forms, with or without |
||||
# modification, are permitted provided that the following conditions are met: |
||||
# * Redistributions of source code must retain the above copyright |
||||
# notice, this list of conditions and the following disclaimer. |
||||
# * Redistributions in binary form must reproduce the above copyright |
||||
# notice, this list of conditions and the following disclaimer in the |
||||
# documentation and/or other materials provided with the distribution. |
||||
# * Neither the name of Google LLC nor the |
||||
# names of its contributors may be used to endorse or promote products |
||||
# derived from this software without specific prior written permission. |
||||
# |
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||||
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY |
||||
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||||
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||||
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
|
||||
load("//bazel:py_proto_library.bzl", "py_proto_library") |
||||
load(":dist.bzl", "py_dist", "py_dist_module") |
||||
load("@bazel_skylib//lib:selects.bzl", "selects") |
||||
load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_VERSION") |
||||
load("@rules_python//python:packaging.bzl", "py_wheel") |
||||
|
||||
licenses(["notice"]) |
||||
|
||||
py_dist_module( |
||||
name = "message_mod", |
||||
extension = "//python:_message_binary", |
||||
module_name = "google.protobuf.pyext._message", |
||||
) |
||||
|
||||
py_dist_module( |
||||
name = "api_implementation_mod", |
||||
extension = "//python:_api_implementation_binary", |
||||
module_name = "google.protobuf.internal.api_implementation", |
||||
) |
||||
|
||||
py_proto_library( |
||||
name = "well_known_proto_py_pb2", |
||||
deps = [ |
||||
"@com_google_protobuf//:any_proto", |
||||
"@com_google_protobuf//:api_proto", |
||||
"@com_google_protobuf//:compiler_plugin_proto", |
||||
"@com_google_protobuf//:descriptor_proto", |
||||
"@com_google_protobuf//:duration_proto", |
||||
"@com_google_protobuf//:empty_proto", |
||||
"@com_google_protobuf//:field_mask_proto", |
||||
"@com_google_protobuf//:source_context_proto", |
||||
"@com_google_protobuf//:struct_proto", |
||||
"@com_google_protobuf//:timestamp_proto", |
||||
"@com_google_protobuf//:type_proto", |
||||
"@com_google_protobuf//:wrappers_proto", |
||||
], |
||||
) |
||||
|
||||
config_setting( |
||||
name = "aarch64_cpu", |
||||
values = {"cpu": "linux-aarch_64"}, |
||||
) |
||||
|
||||
config_setting( |
||||
name = "x86_64_cpu", |
||||
values = {"cpu": "linux-x86_64"}, |
||||
) |
||||
|
||||
config_setting( |
||||
name = "osx-x86_64_cpu", |
||||
values = {"cpu": "osx-x86_64"}, |
||||
) |
||||
|
||||
config_setting( |
||||
name = "win32_cpu", |
||||
values = {"cpu": "win32"}, |
||||
) |
||||
|
||||
config_setting( |
||||
name = "win64_cpu", |
||||
values = {"cpu": "win64"}, |
||||
) |
||||
|
||||
py_wheel( |
||||
name = "binary_wheel", |
||||
abi = select({ |
||||
"//python:full_api_3.7": "cp37m", |
||||
"//python:full_api_3.8": "cp38", |
||||
"//python:full_api_3.9": "cp39", |
||||
"//conditions:default": "abi3", |
||||
}), |
||||
distribution = "protobuf", |
||||
platform = select({ |
||||
":x86_64_cpu": "manylinux2014_x86_64", |
||||
":aarch64_cpu": "manylinux2014_aarch64", |
||||
":osx-x86_64_cpu": "macosx_10_9_universal", |
||||
":win32_cpu": "win32", |
||||
":win64_cpu": "win_amd64", |
||||
"//conditions:default": "any", |
||||
}), |
||||
python_tag = selects.with_or({ |
||||
("//python:limited_api_3.7", "//python:full_api_3.7"): "cp37", |
||||
"//python:full_api_3.8": "cp38", |
||||
"//python:full_api_3.9": "cp39", |
||||
"//python:limited_api_3.10": "cp310", |
||||
"//conditions:default": "system", |
||||
}), |
||||
strip_path_prefixes = ["python/"], |
||||
version = PROTOBUF_VERSION, |
||||
deps = [ |
||||
":message_mod", |
||||
":api_implementation_mod", |
||||
":well_known_proto_py_pb2", |
||||
#TODO(https://github.com/protocolbuffers/upb/issues/503): currently |
||||
# this includes the unit tests. We should filter these out so we are |
||||
# only distributing true source files. |
||||
"@com_google_protobuf//:python_srcs", |
||||
], |
||||
) |
||||
|
||||
py_dist( |
||||
name = "dist", |
||||
binary_wheel = ":binary_wheel", |
||||
full_api_cpus = [ |
||||
"win32", |
||||
"win64", |
||||
], |
||||
# Windows needs version-specific wheels until 3.10. |
||||
full_api_versions = [ |
||||
"37", |
||||
"38", |
||||
"39", |
||||
], |
||||
# Limited API: these wheels will satisfy any Python version >= the |
||||
# given version. |
||||
# |
||||
# Technically the limited API doesn't have the functions we need until |
||||
# 3.10, but on Linux we can get away with using 3.7 (see ../python_api.h for |
||||
# details). |
||||
limited_api_wheels = { |
||||
"win32": "310", |
||||
"win64": "310", |
||||
"linux-x86_64": "37", |
||||
"linux-aarch_64": "37", |
||||
}, |
||||
tags = ["manual"], |
||||
) |
||||
@ -0,0 +1,145 @@ |
||||
"""Rules to create python distribution files and properly name them""" |
||||
|
||||
load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") |
||||
load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") |
||||
|
||||
def _get_suffix(limited_api, python_version, cpu): |
||||
suffix = "pyd" if ("win" in cpu) else "so" |
||||
|
||||
if limited_api == True: |
||||
if "win" not in cpu: |
||||
suffix = "abi3." + suffix |
||||
return "." + suffix |
||||
|
||||
if "win32" in cpu or "win64" in cpu: |
||||
if "win32" in cpu: |
||||
abi = "win32" |
||||
elif "win64" in cpu: |
||||
abi = "win_amd64" |
||||
else: |
||||
fail("Unsupported CPU: " + cpu) |
||||
return ".cp{}-{}.{}".format(python_version, abi, suffix) |
||||
|
||||
if python_version == "system": |
||||
python_version = SYSTEM_PYTHON_VERSION |
||||
if int(python_version) < 38: |
||||
python_version += "m" |
||||
abis = { |
||||
"darwin": "darwin", |
||||
"osx-x86_64": "darwin", |
||||
"osx-aarch_64": "darwin", |
||||
"linux-aarch_64": "aarch64-linux-gnu", |
||||
"linux-x86_64": "x86_64-linux-gnu", |
||||
"k8": "x86_64-linux-gnu", |
||||
} |
||||
|
||||
return ".cpython-{}-{}.{}".format(python_version, abis[cpu], suffix) |
||||
|
||||
fail("Unsupported combination of flags") |
||||
|
||||
def _py_dist_module_impl(ctx): |
||||
base_filename = ctx.attr.module_name.replace(".", "/") |
||||
suffix = _get_suffix( |
||||
limited_api = ctx.attr._limited_api[BuildSettingInfo].value, |
||||
python_version = ctx.attr._python_version[BuildSettingInfo].value, |
||||
cpu = ctx.var["TARGET_CPU"], |
||||
) |
||||
filename = base_filename + suffix |
||||
file = ctx.actions.declare_file(filename) |
||||
src = ctx.attr.extension[DefaultInfo].files.to_list()[0] |
||||
ctx.actions.run( |
||||
executable = "cp", |
||||
arguments = [src.path, file.path], |
||||
inputs = [src], |
||||
outputs = [file], |
||||
) |
||||
return [ |
||||
DefaultInfo(files = depset([file])), |
||||
] |
||||
|
||||
_py_dist_module_rule = rule( |
||||
output_to_genfiles = True, |
||||
implementation = _py_dist_module_impl, |
||||
fragments = ["cpp"], |
||||
attrs = { |
||||
"module_name": attr.string(mandatory = True), |
||||
"extension": attr.label( |
||||
mandatory = True, |
||||
providers = [CcInfo], |
||||
), |
||||
"_limited_api": attr.label(default = "//python:limited_api"), |
||||
"_python_version": attr.label(default = "//python:python_version"), |
||||
"_cc_toolchain": attr.label( |
||||
default = "@bazel_tools//tools/cpp:current_cc_toolchain", |
||||
), |
||||
}, |
||||
) |
||||
|
||||
def py_dist_module(name, module_name, extension): |
||||
file_rule = name + "_file" |
||||
_py_dist_module_rule( |
||||
name = file_rule, |
||||
module_name = module_name, |
||||
extension = extension, |
||||
) |
||||
|
||||
# TODO(haberman): needed? |
||||
native.py_library( |
||||
name = name, |
||||
data = [":" + file_rule], |
||||
imports = ["."], |
||||
) |
||||
|
||||
def _py_dist_transition_impl(settings, attr): |
||||
_ignore = (settings) # @unused |
||||
transitions = [] |
||||
|
||||
for cpu, version in attr.limited_api_wheels.items(): |
||||
transitions.append({ |
||||
"//command_line_option:cpu": cpu, |
||||
"//python:python_version": version, |
||||
"//python:limited_api": True, |
||||
}) |
||||
|
||||
for version in attr.full_api_versions: |
||||
for cpu in attr.full_api_cpus: |
||||
transitions.append({ |
||||
"//command_line_option:cpu": cpu, |
||||
"//python:python_version": version, |
||||
"//python:limited_api": False, |
||||
}) |
||||
|
||||
return transitions |
||||
|
||||
_py_dist_transition = transition( |
||||
implementation = _py_dist_transition_impl, |
||||
inputs = [], |
||||
outputs = [ |
||||
"//command_line_option:cpu", |
||||
"//python:python_version", |
||||
"//python:limited_api", |
||||
], |
||||
) |
||||
|
||||
def _py_dist_impl(ctx): |
||||
return [ |
||||
DefaultInfo(files = depset( |
||||
transitive = [dep[DefaultInfo].files for dep in ctx.attr.binary_wheel], |
||||
)), |
||||
] |
||||
|
||||
py_dist = rule( |
||||
implementation = _py_dist_impl, |
||||
attrs = { |
||||
"binary_wheel": attr.label( |
||||
mandatory = True, |
||||
cfg = _py_dist_transition, |
||||
), |
||||
"limited_api_wheels": attr.string_dict(), |
||||
"full_api_versions": attr.string_list(), |
||||
"full_api_cpus": attr.string_list(), |
||||
"_allowlist_function_transition": attr.label( |
||||
default = "@bazel_tools//tools/allowlists/function_transition_allowlist", |
||||
), |
||||
}, |
||||
) |
||||
@ -0,0 +1,70 @@ |
||||
"""Macro to support py_extension |
||||
""" |
||||
|
||||
def py_extension(name, srcs, copts, deps = []): |
||||
"""Creates a C++ library to extend python |
||||
|
||||
Args: |
||||
name: Name of the target |
||||
srcs: List of source files to create the target |
||||
copts: List of C++ compile options to use |
||||
deps: Libraries that the target depends on |
||||
""" |
||||
|
||||
version_script = name + "_version_script.lds" |
||||
symbol = "PyInit_" + name |
||||
native.genrule( |
||||
name = "gen_" + version_script, |
||||
outs = [version_script], |
||||
cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@", |
||||
) |
||||
|
||||
native.cc_binary( |
||||
name = name + "_binary", |
||||
srcs = srcs, |
||||
copts = copts, |
||||
linkopts = select({ |
||||
"//python/dist:osx-x86_64_cpu": ["-undefined", "dynamic_lookup"], |
||||
"//conditions:default": [], |
||||
}), |
||||
linkshared = True, |
||||
linkstatic = True, |
||||
deps = deps + [ |
||||
":" + version_script, |
||||
] + select({ |
||||
"//python:limited_api_3.7": ["@python-3.7.0//:python_headers"], |
||||
"//python:full_api_3.7_win32": ["@nuget_python_i686_3.7.0//:python_full_api"], |
||||
"//python:full_api_3.7_win64": ["@nuget_python_x86-64_3.7.0//:python_full_api"], |
||||
"//python:full_api_3.8_win32": ["@nuget_python_i686_3.8.0//:python_full_api"], |
||||
"//python:full_api_3.8_win64": ["@nuget_python_x86-64_3.8.0//:python_full_api"], |
||||
"//python:full_api_3.9_win32": ["@nuget_python_i686_3.9.0//:python_full_api"], |
||||
"//python:full_api_3.9_win64": ["@nuget_python_x86-64_3.9.0//:python_full_api"], |
||||
"//python:limited_api_3.10_win32": ["@nuget_python_i686_3.10.0//:python_limited_api"], |
||||
"//python:limited_api_3.10_win64": ["@nuget_python_x86-64_3.10.0//:python_limited_api"], |
||||
"//conditions:default": ["@system_python//:python_headers"], |
||||
}), |
||||
) |
||||
|
||||
EXT_SUFFIX = ".abi3.so" |
||||
|
||||
module_name_map = { |
||||
"_message": "pyext", |
||||
"_api_implementation": "internal", |
||||
} |
||||
|
||||
output_file = "google/protobuf/" + module_name_map[name] + "/" + name + EXT_SUFFIX |
||||
|
||||
native.genrule( |
||||
name = "copy" + name, |
||||
srcs = [":" + name + "_binary"], |
||||
outs = [output_file], |
||||
cmd = "cp $< $@", |
||||
visibility = ["//python:__subpackages__"], |
||||
) |
||||
|
||||
native.py_library( |
||||
name = name, |
||||
data = [output_file], |
||||
imports = ["."], |
||||
visibility = ["//python:__subpackages__"], |
||||
) |
||||
Loading…
Reference in new issue