From ee6b1abb35ed852201937402a65be4b866849fec Mon Sep 17 00:00:00 2001 From: Protobuf Team Date: Wed, 13 Apr 2022 09:06:17 -0700 Subject: [PATCH] Create targets for UPB release PiperOrigin-RevId: 441496547 --- WORKSPACE | 42 ++- bazel/py_extension.bzl | 38 --- bazel/pyproto_test_wrapper.bzl | 3 +- bazel/python_downloads.bzl | 84 ++++++ .../{workspace_defs.bzl => system_python.bzl} | 12 +- bazel/workspace_deps.bzl | 2 +- cmake/make_cmakelists.py | 6 + python/BUILD | 250 +++++++++++------- python/api_implementation.c | 2 +- python/descriptor.h | 2 +- python/dist/BUILD.bazel | 152 +++++++++++ python/dist/dist.bzl | 145 ++++++++++ python/extension_dict.h | 2 +- python/map.h | 2 +- python/protobuf.h | 2 +- python/py_extension.bzl | 70 +++++ python/{python.h => python_api.h} | 30 ++- python/repeated.h | 2 +- 18 files changed, 697 insertions(+), 149 deletions(-) delete mode 100644 bazel/py_extension.bzl create mode 100644 bazel/python_downloads.bzl rename bazel/{workspace_defs.bzl => system_python.bzl} (86%) create mode 100644 python/dist/BUILD.bazel create mode 100644 python/dist/dist.bzl create mode 100644 python/py_extension.bzl rename python/{python.h => python_api.h} (61%) diff --git a/WORKSPACE b/WORKSPACE index 0bd8479ecc..823037bc8c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,9 @@ workspace(name = "upb") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//bazel:python_downloads.bzl", "python_source_archive", "python_nuget_package") load("//bazel:workspace_deps.bzl", "upb_deps") -load("//bazel:workspace_defs.bzl", "system_python") +load("//bazel:system_python.bzl", "system_python") upb_deps() @@ -62,3 +63,42 @@ rules_fuzzing_dependencies() load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") rules_fuzzing_init() + +#Python Downloads + +python_source_archive( + name = "python-3.7.0", + sha256 = "85bb9feb6863e04fb1700b018d9d42d1caac178559ffa453d7e6a436e259fd0d", +) +python_nuget_package( + name = "nuget_python_i686_3.7.0", + sha256 = "a8bb49fa1ca62ad55430fcafaca1b58015e22943e66b1a87d5e7cef2556c6a54", +) +python_nuget_package( + name = "nuget_python_x86-64_3.7.0", + sha256 = "66eb796a5bdb1e6787b8f655a1237a6b6964af2115b7627cf4f0032cf068b4b2", +) +python_nuget_package( + name = "nuget_python_i686_3.8.0", + sha256 = "87a6481f5eef30b42ac12c93f06f73bd0b8692f26313b76a6615d1641c4e7bca", +) +python_nuget_package( + name = "nuget_python_x86-64_3.8.0", + sha256 = "96c61321ce90dd053c8a04f305a5f6cc6d91350b862db34440e4a4f069b708a0", +) +python_nuget_package( + name = "nuget_python_i686_3.9.0", + sha256 = "229abecbe49dc08fe5709e0b31e70edfb3b88f23335ebfc2904c44f940fd59b6", +) +python_nuget_package( + name = "nuget_python_x86-64_3.9.0", + sha256 = "6af58a733e7dfbfcdd50d55788134393d6ffe7ab8270effbf724bdb786558832", +) +python_nuget_package( + name = "nuget_python_i686_3.10.0", + sha256 = "e115e102eb90ce160ab0ef7506b750a8d7ecc385bde0a496f02a54337a8bc333", +) +python_nuget_package( + name = "nuget_python_x86-64_3.10.0", + sha256 = "4474c83c25625d93e772e926f95f4cd398a0abbb52793625fa30f39af3d2cc00", +) diff --git a/bazel/py_extension.bzl b/bazel/py_extension.bzl deleted file mode 100644 index eacd39b856..0000000000 --- a/bazel/py_extension.bzl +++ /dev/null @@ -1,38 +0,0 @@ -load( - "//bazel:build_defs.bzl", - "UPB_DEFAULT_COPTS", -) - -def py_extension(name, srcs, deps = []): - version_script = name + "_version_script.lds" - symbol = "PyInit_" + name - native.genrule( - name = "gen_" + version_script, - outs = [version_script], - cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@", - ) - - native.cc_binary( - name = name, - srcs = srcs, - copts = UPB_DEFAULT_COPTS + [ - # The Python API requires patterns that are ISO C incompatible, like - # casts between function pointers and object pointers. - "-Wno-pedantic", - ], - # We use a linker script to hide all symbols except the entry point for - # the module. - linkopts = select({ - "@platforms//os:linux": ["-Wl,--version-script,$(location :" + version_script + ")"], - "@platforms//os:macos": [ - "-Wl,-exported_symbol", - "-Wl,_" + symbol, - ], - }), - linkshared = True, - linkstatic = True, - deps = deps + [ - ":" + version_script, - "@system_python//:python_headers", - ], - ) diff --git a/bazel/pyproto_test_wrapper.bzl b/bazel/pyproto_test_wrapper.bzl index 629ee9b2c6..309231467e 100644 --- a/bazel/pyproto_test_wrapper.bzl +++ b/bazel/pyproto_test_wrapper.bzl @@ -9,7 +9,8 @@ def pyproto_test_wrapper(name): main = src, data = ["@com_google_protobuf//:testdata"], deps = [ - "//python:message_ext", + "//python:_api_implementation", + "//python:_message", "@com_google_protobuf//:python_common_test_protos", "@com_google_protobuf//:python_specific_test_protos", "@com_google_protobuf//:python_srcs", diff --git a/bazel/python_downloads.bzl b/bazel/python_downloads.bzl new file mode 100644 index 0000000000..e237c93631 --- /dev/null +++ b/bazel/python_downloads.bzl @@ -0,0 +1,84 @@ +"""Helper methods to download different python versions""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +limited_api_build_file = """ +cc_library( + name = "python_headers", + hdrs = glob(["**/Include/**/*.h"]), + strip_include_prefix = "Python-{}/Include", + visibility = ["//visibility:public"], +) +""" + +def python_source_archive(name, sha256): + """Helper method to create a python_headers target that will work for linux and macos. + + Args: + name: The name of the target, should be in the form python_{VERSION} + sha256: The sha256 of the python package for the specified version + """ + version = name.split("-")[1] + http_archive( + name = name, + urls = [ + "https://www.python.org/ftp/python/{0}/Python-{0}.tgz" + .format(version), + ], + sha256 = sha256, + build_file_content = limited_api_build_file.format(version), + patch_cmds = [ + "echo '#define SIZEOF_WCHAR_T 4' > Python-{}/Include/pyconfig.h" + .format(version), + ], + ) + +nuget_build_file = """ +cc_import( + name = "python_full_api", + hdrs = glob(["**/*.h"]), + shared_library = "python{0}.dll", + interface_library = "libs/python{0}.lib", + visibility = ["@upb//python:__pkg__"], +) + +cc_import( + name = "python_limited_api", + hdrs = glob(["**/*.h"]), + shared_library = "python{1}.dll", + interface_library = "libs/python{1}.lib", + visibility = ["@upb//python:__pkg__"], +) +""" + +def python_nuget_package(name, sha256): + """Helper method to create full and limited api dependencies for windows using nuget + + Args: + name: The name of the target, should be in the form nuget_python_{CPU}_{VERSION} + sha256: The sha256 of the nuget package for that version + """ + cpu = name.split("_")[2] + version = name.split("_")[3] + + full_api_lib_number = version.split(".")[0] + version.split(".")[1] + limited_api_lib_number = version.split(".")[0] + + folder_name_dict = { + "i686": "pythonx86", + "x86-64": "python", + } + + http_archive( + name = name, + urls = [ + "https://www.nuget.org/api/v2/package/{}/{}" + .format(folder_name_dict[cpu], version), + ], + sha256 = sha256, + strip_prefix = "tools", + build_file_content = + nuget_build_file.format(full_api_lib_number, limited_api_lib_number), + type = "zip", + patch_cmds = ["cp -r include/* ."], + ) diff --git a/bazel/workspace_defs.bzl b/bazel/system_python.bzl similarity index 86% rename from bazel/workspace_defs.bzl rename to bazel/system_python.bzl index 03655b2409..9e20b1fdd8 100644 --- a/bazel/workspace_defs.bzl +++ b/bazel/system_python.bzl @@ -27,6 +27,7 @@ _build_file = """ load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") + cc_library( name = "python_headers", hdrs = glob(["python/**/*.h"]), @@ -36,7 +37,7 @@ cc_library( py_runtime( name = "py3_runtime", - interpreter_path = "%s", + interpreter_path = "{}", python_version = "PY3", ) @@ -52,6 +53,11 @@ toolchain( ) """ +def _get_python_version(repository_ctx): + py_program = "import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))" + result = repository_ctx.execute(["python3", "-c", py_program]) + return (result.stdout).strip() + def _get_config_var(repository_ctx, name): py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')" result = repository_ctx.execute(["python3", "-c", py_program % (name)]) @@ -63,7 +69,9 @@ def _python_headers_impl(repository_ctx): path = _get_config_var(repository_ctx, "INCLUDEPY") repository_ctx.symlink(path, "python") python3 = repository_ctx.which("python3") - repository_ctx.file("BUILD.bazel", _build_file % python3) + python_version = _get_python_version(repository_ctx) + repository_ctx.file("BUILD.bazel", _build_file.format(python3)) + repository_ctx.file("version.bzl", "SYSTEM_PYTHON_VERSION = {}".format(python_version)) # The system_python() repository rule exposes Python headers from the system. # diff --git a/bazel/workspace_deps.bzl b/bazel/workspace_deps.bzl index db3afd7a79..ca9b756990 100644 --- a/bazel/workspace_deps.bzl +++ b/bazel/workspace_deps.bzl @@ -14,7 +14,7 @@ def upb_deps(): maybe( git_repository, name = "com_google_protobuf", - commit = "2f91da585e96a7efe43505f714f03c7716a94ecb", + commit = "a69354f31b253856689ae765a9ea3217ec001873", remote = "https://github.com/protocolbuffers/protobuf.git", patches = [ "//bazel:protobuf.patch", diff --git a/cmake/make_cmakelists.py b/cmake/make_cmakelists.py index 29c73bb8bc..655d8095c0 100755 --- a/cmake/make_cmakelists.py +++ b/cmake/make_cmakelists.py @@ -234,6 +234,12 @@ class WorkspaceFileFunctions(object): def register_toolchains(self, toolchain): pass + def python_source_archive(self, **kwargs): + pass + + def python_nuget_package(self, **kwargs): + pass + class Converter(object): def __init__(self): diff --git a/python/BUILD b/python/BUILD index ac971132b5..ac96b6c66e 100644 --- a/python/BUILD +++ b/python/BUILD @@ -24,13 +24,154 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # begin:github_only -load("//bazel:py_proto_library.bzl", "py_proto_library") -load("//bazel:py_extension.bzl", "py_extension") -load("@rules_python//python:packaging.bzl", "py_wheel") +load("//python:py_extension.bzl", "py_extension") # end:github_only +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") +load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") + licenses(["notice"]) +package( + default_visibility = ["//python/dist:__pkg__"], +) + +LIMITED_API_FLAG_SELECT = { + ":limited_api_3.7": ["-DPy_LIMITED_API=0x03070000"], + ":limited_api_3.10": ["-DPy_LIMITED_API=0x030a0000"], + "//conditions:default": [], +} + +bool_flag( + name = "limited_api", + build_setting_default = False, +) + +string_flag( + name = "python_version", + build_setting_default = "system", + values = [ + "system", + "37", + "38", + "39", + "310", + ], +) + +config_setting( + name = "limited_api_3.7", + flag_values = { + ":limited_api": "True", + ":python_version": "37", + }, +) + +config_setting( + name = "full_api_3.7_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "37", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.7_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "37", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.7", + match_any = [ + ":full_api_3.7_win32", + ":full_api_3.7_win64", + ], +) + +config_setting( + name = "full_api_3.8_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "38", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.8_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "38", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.8", + match_any = [ + ":full_api_3.8_win32", + ":full_api_3.8_win64", + ], +) + +config_setting( + name = "full_api_3.9_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "39", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.9_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "39", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.9", + match_any = [ + "full_api_3.9_win32", + ":full_api_3.9_win64", + ], +) + +config_setting( + name = "limited_api_3.10_win32", + flag_values = { + ":limited_api": "True", + ":python_version": "310", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "limited_api_3.10_win64", + flag_values = { + ":limited_api": "True", + ":python_version": "310", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "limited_api_3.10", + match_any = [ + ":limited_api_3.10_win32", + ":limited_api_3.10_win64", + ], +) + py_extension( name = "_message", srcs = [ @@ -50,10 +191,15 @@ py_extension( "message.h", "protobuf.c", "protobuf.h", - "python.h", + "python_api.h", "repeated.c", "repeated.h", ], + copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [ + # The Python API requires patterns that are ISO C incompatible, like + # casts between function pointers and object pointers. + "-Wno-pedantic", + ], deps = [ "//:descriptor_upb_proto_reflection", "//:reflection", @@ -68,97 +214,13 @@ py_extension( py_extension( name = "_api_implementation", - srcs = ["api_implementation.c"], -) - -# begin:github_only - -py_test( - name = "minimal_test", srcs = [ - "minimal_test.py", + "api_implementation.c", + "python_api.h", ], - imports = ["."], - legacy_create_init = False, - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", + copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [ + # The Python API requires patterns that are ISO C incompatible, like + # casts between function pointers and object pointers. + "-Wno-pedantic", ], ) - -# Copy the extensions into the location recognized by Python. -# .abi3.so indicates use of the limited API, and cross-version ABI compatibility. -EXT_SUFFIX = ".abi3.so" - -genrule( - name = "copy_message", - srcs = [":_message"], - outs = ["google/protobuf/pyext/_message" + EXT_SUFFIX], - cmd = "cp $< $@", -) - -genrule( - name = "copy_api_implementation", - srcs = [":_api_implementation"], - outs = ["google/protobuf/internal/_api_implementation" + EXT_SUFFIX], - cmd = "cp $< $@", - visibility = ["//python:__subpackages__"], -) - -filegroup( - name = "extension_files", - srcs = [ - "google/protobuf/pyext/_message" + EXT_SUFFIX, - "google/protobuf/internal/_api_implementation" + EXT_SUFFIX, - ], -) - -py_library( - name = "message_ext", - data = [":extension_files"], - imports = ["."], - visibility = ["//python:__subpackages__"], -) - -py_proto_library( - name = "well_known_proto_pb2", - deps = [ - "@com_google_protobuf//:any_proto", - "@com_google_protobuf//:api_proto", - "@com_google_protobuf//:compiler_plugin_proto", - "@com_google_protobuf//:descriptor_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:empty_proto", - "@com_google_protobuf//:field_mask_proto", - "@com_google_protobuf//:source_context_proto", - "@com_google_protobuf//:struct_proto", - "@com_google_protobuf//:timestamp_proto", - "@com_google_protobuf//:type_proto", - "@com_google_protobuf//:wrappers_proto", - ], -) - -py_wheel( - name = "binary_wheel", - abi = "abi3", - distribution = "protobuf", - # TODO(https://github.com/protocolbuffers/upb/issues/502): we need to make - # this a select() that is calculated from the platform we are actually - # building on. - platform = "manylinux2014_x86_64", - python_tag = "cp36", - strip_path_prefixes = ["python/"], - version = "4.20.0", - deps = [ - ":extension_files", - ":well_known_proto_pb2", - # TODO(https://github.com/protocolbuffers/upb/issues/503): currently - # this includes the unit tests. We should filter these out so we are - # only distributing true source files. - "@com_google_protobuf//:python_srcs", - ], -) - -# end:github_only diff --git a/python/api_implementation.c b/python/api_implementation.c index 9f96b936d3..744f2beb71 100644 --- a/python/api_implementation.c +++ b/python/api_implementation.c @@ -25,7 +25,7 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include +#include "python/python_api.h" static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, diff --git a/python/descriptor.h b/python/descriptor.h index 6cb6c1c293..581e1c7a8c 100644 --- a/python/descriptor.h +++ b/python/descriptor.h @@ -30,7 +30,7 @@ #include -#include "python/python.h" +#include "python/python_api.h" #include "upb/def.h" typedef enum { diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel new file mode 100644 index 0000000000..33e6287432 --- /dev/null +++ b/python/dist/BUILD.bazel @@ -0,0 +1,152 @@ +# Copyright (c) 2009-2022, Google LLC +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Google LLC nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +load("//bazel:py_proto_library.bzl", "py_proto_library") +load(":dist.bzl", "py_dist", "py_dist_module") +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_VERSION") +load("@rules_python//python:packaging.bzl", "py_wheel") + +licenses(["notice"]) + +py_dist_module( + name = "message_mod", + extension = "//python:_message_binary", + module_name = "google.protobuf.pyext._message", +) + +py_dist_module( + name = "api_implementation_mod", + extension = "//python:_api_implementation_binary", + module_name = "google.protobuf.internal.api_implementation", +) + +py_proto_library( + name = "well_known_proto_py_pb2", + deps = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", + ], +) + +config_setting( + name = "aarch64_cpu", + values = {"cpu": "linux-aarch_64"}, +) + +config_setting( + name = "x86_64_cpu", + values = {"cpu": "linux-x86_64"}, +) + +config_setting( + name = "osx-x86_64_cpu", + values = {"cpu": "osx-x86_64"}, +) + +config_setting( + name = "win32_cpu", + values = {"cpu": "win32"}, +) + +config_setting( + name = "win64_cpu", + values = {"cpu": "win64"}, +) + +py_wheel( + name = "binary_wheel", + abi = select({ + "//python:full_api_3.7": "cp37m", + "//python:full_api_3.8": "cp38", + "//python:full_api_3.9": "cp39", + "//conditions:default": "abi3", + }), + distribution = "protobuf", + platform = select({ + ":x86_64_cpu": "manylinux2014_x86_64", + ":aarch64_cpu": "manylinux2014_aarch64", + ":osx-x86_64_cpu": "macosx_10_9_universal", + ":win32_cpu": "win32", + ":win64_cpu": "win_amd64", + "//conditions:default": "any", + }), + python_tag = selects.with_or({ + ("//python:limited_api_3.7", "//python:full_api_3.7"): "cp37", + "//python:full_api_3.8": "cp38", + "//python:full_api_3.9": "cp39", + "//python:limited_api_3.10": "cp310", + "//conditions:default": "system", + }), + strip_path_prefixes = ["python/"], + version = PROTOBUF_VERSION, + deps = [ + ":message_mod", + ":api_implementation_mod", + ":well_known_proto_py_pb2", + #TODO(https://github.com/protocolbuffers/upb/issues/503): currently + # this includes the unit tests. We should filter these out so we are + # only distributing true source files. + "@com_google_protobuf//:python_srcs", + ], +) + +py_dist( + name = "dist", + binary_wheel = ":binary_wheel", + full_api_cpus = [ + "win32", + "win64", + ], + # Windows needs version-specific wheels until 3.10. + full_api_versions = [ + "37", + "38", + "39", + ], + # Limited API: these wheels will satisfy any Python version >= the + # given version. + # + # Technically the limited API doesn't have the functions we need until + # 3.10, but on Linux we can get away with using 3.7 (see ../python_api.h for + # details). + limited_api_wheels = { + "win32": "310", + "win64": "310", + "linux-x86_64": "37", + "linux-aarch_64": "37", + }, + tags = ["manual"], +) diff --git a/python/dist/dist.bzl b/python/dist/dist.bzl new file mode 100644 index 0000000000..6760a381b3 --- /dev/null +++ b/python/dist/dist.bzl @@ -0,0 +1,145 @@ +"""Rules to create python distribution files and properly name them""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") + +def _get_suffix(limited_api, python_version, cpu): + suffix = "pyd" if ("win" in cpu) else "so" + + if limited_api == True: + if "win" not in cpu: + suffix = "abi3." + suffix + return "." + suffix + + if "win32" in cpu or "win64" in cpu: + if "win32" in cpu: + abi = "win32" + elif "win64" in cpu: + abi = "win_amd64" + else: + fail("Unsupported CPU: " + cpu) + return ".cp{}-{}.{}".format(python_version, abi, suffix) + + if python_version == "system": + python_version = SYSTEM_PYTHON_VERSION + if int(python_version) < 38: + python_version += "m" + abis = { + "darwin": "darwin", + "osx-x86_64": "darwin", + "osx-aarch_64": "darwin", + "linux-aarch_64": "aarch64-linux-gnu", + "linux-x86_64": "x86_64-linux-gnu", + "k8": "x86_64-linux-gnu", + } + + return ".cpython-{}-{}.{}".format(python_version, abis[cpu], suffix) + + fail("Unsupported combination of flags") + +def _py_dist_module_impl(ctx): + base_filename = ctx.attr.module_name.replace(".", "/") + suffix = _get_suffix( + limited_api = ctx.attr._limited_api[BuildSettingInfo].value, + python_version = ctx.attr._python_version[BuildSettingInfo].value, + cpu = ctx.var["TARGET_CPU"], + ) + filename = base_filename + suffix + file = ctx.actions.declare_file(filename) + src = ctx.attr.extension[DefaultInfo].files.to_list()[0] + ctx.actions.run( + executable = "cp", + arguments = [src.path, file.path], + inputs = [src], + outputs = [file], + ) + return [ + DefaultInfo(files = depset([file])), + ] + +_py_dist_module_rule = rule( + output_to_genfiles = True, + implementation = _py_dist_module_impl, + fragments = ["cpp"], + attrs = { + "module_name": attr.string(mandatory = True), + "extension": attr.label( + mandatory = True, + providers = [CcInfo], + ), + "_limited_api": attr.label(default = "//python:limited_api"), + "_python_version": attr.label(default = "//python:python_version"), + "_cc_toolchain": attr.label( + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + ), + }, +) + +def py_dist_module(name, module_name, extension): + file_rule = name + "_file" + _py_dist_module_rule( + name = file_rule, + module_name = module_name, + extension = extension, + ) + + # TODO(haberman): needed? + native.py_library( + name = name, + data = [":" + file_rule], + imports = ["."], + ) + +def _py_dist_transition_impl(settings, attr): + _ignore = (settings) # @unused + transitions = [] + + for cpu, version in attr.limited_api_wheels.items(): + transitions.append({ + "//command_line_option:cpu": cpu, + "//python:python_version": version, + "//python:limited_api": True, + }) + + for version in attr.full_api_versions: + for cpu in attr.full_api_cpus: + transitions.append({ + "//command_line_option:cpu": cpu, + "//python:python_version": version, + "//python:limited_api": False, + }) + + return transitions + +_py_dist_transition = transition( + implementation = _py_dist_transition_impl, + inputs = [], + outputs = [ + "//command_line_option:cpu", + "//python:python_version", + "//python:limited_api", + ], +) + +def _py_dist_impl(ctx): + return [ + DefaultInfo(files = depset( + transitive = [dep[DefaultInfo].files for dep in ctx.attr.binary_wheel], + )), + ] + +py_dist = rule( + implementation = _py_dist_impl, + attrs = { + "binary_wheel": attr.label( + mandatory = True, + cfg = _py_dist_transition, + ), + "limited_api_wheels": attr.string_dict(), + "full_api_versions": attr.string_list(), + "full_api_cpus": attr.string_list(), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) diff --git a/python/extension_dict.h b/python/extension_dict.h index f8a8fc43ac..a21822c0ee 100644 --- a/python/extension_dict.h +++ b/python/extension_dict.h @@ -30,7 +30,7 @@ #include -#include "python/python.h" +#include "python/python_api.h" PyObject* PyUpb_ExtensionDict_New(PyObject* msg); diff --git a/python/map.h b/python/map.h index 0c03ed06dd..aaa4e20385 100644 --- a/python/map.h +++ b/python/map.h @@ -30,7 +30,7 @@ #include -#include "python/python.h" +#include "python/python_api.h" #include "upb/def.h" // Creates a new repeated field stub for field `f` of message object `parent`. diff --git a/python/protobuf.h b/python/protobuf.h index 0b87646895..262f33e28c 100644 --- a/python/protobuf.h +++ b/python/protobuf.h @@ -31,7 +31,7 @@ #include #include "python/descriptor.h" -#include "python/python.h" +#include "python/python_api.h" #include "upb/table_internal.h" diff --git a/python/py_extension.bzl b/python/py_extension.bzl new file mode 100644 index 0000000000..2fea049a0d --- /dev/null +++ b/python/py_extension.bzl @@ -0,0 +1,70 @@ +"""Macro to support py_extension +""" + +def py_extension(name, srcs, copts, deps = []): + """Creates a C++ library to extend python + + Args: + name: Name of the target + srcs: List of source files to create the target + copts: List of C++ compile options to use + deps: Libraries that the target depends on + """ + + version_script = name + "_version_script.lds" + symbol = "PyInit_" + name + native.genrule( + name = "gen_" + version_script, + outs = [version_script], + cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@", + ) + + native.cc_binary( + name = name + "_binary", + srcs = srcs, + copts = copts, + linkopts = select({ + "//python/dist:osx-x86_64_cpu": ["-undefined", "dynamic_lookup"], + "//conditions:default": [], + }), + linkshared = True, + linkstatic = True, + deps = deps + [ + ":" + version_script, + ] + select({ + "//python:limited_api_3.7": ["@python-3.7.0//:python_headers"], + "//python:full_api_3.7_win32": ["@nuget_python_i686_3.7.0//:python_full_api"], + "//python:full_api_3.7_win64": ["@nuget_python_x86-64_3.7.0//:python_full_api"], + "//python:full_api_3.8_win32": ["@nuget_python_i686_3.8.0//:python_full_api"], + "//python:full_api_3.8_win64": ["@nuget_python_x86-64_3.8.0//:python_full_api"], + "//python:full_api_3.9_win32": ["@nuget_python_i686_3.9.0//:python_full_api"], + "//python:full_api_3.9_win64": ["@nuget_python_x86-64_3.9.0//:python_full_api"], + "//python:limited_api_3.10_win32": ["@nuget_python_i686_3.10.0//:python_limited_api"], + "//python:limited_api_3.10_win64": ["@nuget_python_x86-64_3.10.0//:python_limited_api"], + "//conditions:default": ["@system_python//:python_headers"], + }), + ) + + EXT_SUFFIX = ".abi3.so" + + module_name_map = { + "_message": "pyext", + "_api_implementation": "internal", + } + + output_file = "google/protobuf/" + module_name_map[name] + "/" + name + EXT_SUFFIX + + native.genrule( + name = "copy" + name, + srcs = [":" + name + "_binary"], + outs = [output_file], + cmd = "cp $< $@", + visibility = ["//python:__subpackages__"], + ) + + native.py_library( + name = name, + data = [output_file], + imports = ["."], + visibility = ["//python:__subpackages__"], + ) diff --git a/python/python.h b/python/python_api.h similarity index 61% rename from python/python.h rename to python/python_api.h index 7eef3e5fe0..e8a8fcf818 100644 --- a/python/python.h +++ b/python/python_api.h @@ -28,15 +28,33 @@ #ifndef PYUPB_PYTHON_H__ #define PYUPB_PYTHON_H__ -// We restrict ourselves to the limited API, so that we will be ABI-compatible -// with any version of Python >= 3.6.1 (3.6.1 introduce PySlice_Unpack()) -#define Py_LIMITED_API 0x03060100 -#include +// We restrict ourselves to the limited API, so that a single build can be +// ABI-compatible with a wide range of Python versions. +// +// The build system will define Py_LIMITED_API as appropriate (see BUILD). We +// only want to define it for our distribution packages, since we can do some +// extra assertions when Py_LIMITED_API is not defined. Also Py_LIMITED_API is +// incompatible with Py_DEBUG. -// This function was not officially added to the limited API until Python 3.10. -// But in practice it has been stable since Python 3.1. See: +// #define Py_LIMITED_API // Defined by build system when appropriate. + +#include "Python.h" + +// Ideally we could restrict ourselves to the limited API of 3.7, but this is +// a very important function that was not officially added to the limited API +// until 3.10. Without this function, there is no way of getting data from a +// Python `str` object without a copy. +// +// While this function was not *officially* added to the limited API until +// Python 3.10, In practice it has been stable since Python 3.1. // https://bugs.python.org/issue41784 +// +// On Linux, ELF lets us get away with using this function with the limited +// API prior to 3.10. + +#if defined(__linux__) && defined(Py_LIMITED_API) && Py_LIMITED_API < 0x03100000 PyAPI_FUNC(const char*) PyUnicode_AsUTF8AndSize(PyObject* unicode, Py_ssize_t* size); +#endif #endif // PYUPB_PYTHON_H__ diff --git a/python/repeated.h b/python/repeated.h index 9cde3eae17..5d74bd2edb 100644 --- a/python/repeated.h +++ b/python/repeated.h @@ -30,7 +30,7 @@ #include -#include "python/python.h" +#include "python/python_api.h" #include "upb/def.h" // Creates a new repeated field stub for field `f` of message object `parent`.