Merge pull request #531 from haberman/python-google3

Changes to make Python extension build in google3

BUILD

@@ -444,7 +444,10 @@ cc_library(
         "upb/table_internal.h",
         "upb/upb.h",
     ],
-    visibility = ["//tests:__pkg__"],
+    visibility = [
+        "//python:__pkg__",
+        "//tests:__pkg__",
+    ],
     deps = [
         ":port",
     ],

bazel/py_extension.bzl

@@ -0,0 +1,39 @@
+
+load(
+    "//bazel:build_defs.bzl",
+    "UPB_DEFAULT_COPTS",
+)
+
+def py_extension(name, srcs, deps=[]):
+    version_script = name + "_version_script.lds"
+    symbol = "PyInit_" + name
+    native.genrule(
+        name = "gen_" + version_script,
+        outs = [version_script],
+        cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@",
+    )
+
+    native.cc_binary(
+        name = name,
+        srcs = srcs,
+        copts = UPB_DEFAULT_COPTS + [
+            # The Python API requires patterns that are ISO C incompatible, like
+            # casts between function pointers and object pointers.
+            "-Wno-pedantic",
+        ],
+        # We use a linker script to hide all symbols except the entry point for
+        # the module.
+        linkopts = select({
+            "@platforms//os:linux": ["-Wl,--version-script,$(location :" + version_script + ")"],
+            "@platforms//os:macos": [
+                "-Wl,-exported_symbol",
+                "-Wl,_" + symbol,
+            ],
+        }),
+        linkshared = True,
+        linkstatic = True,
+        deps = deps + [
+            ":" + version_script,
+            "@system_python//:python_headers",
+        ],
+    )

bazel/py_proto_library.bzl

@@ -103,7 +103,7 @@ def _py_proto_library_aspect_impl(target, ctx):
     )
     outs_depset = depset(srcs)
     return [
-        PyInfo(transitive_sources = outs_depset)
+        PyInfo(transitive_sources = outs_depset),
     ]
 
 _py_proto_library_aspect = aspect(

bazel/workspace_defs.bzl

@@ -53,17 +53,17 @@ toolchain(
 """
 
 def _get_config_var(repository_ctx, name):
-  py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')"
+    py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')"
-  result = repository_ctx.execute(["python3", "-c", py_program % (name)])
+    result = repository_ctx.execute(["python3", "-c", py_program % (name)])
-  if result.return_code != 0:
+    if result.return_code != 0:
-    fail("No python3 executable available on the system")
+        fail("No python3 executable available on the system")
-  return result.stdout
+    return result.stdout
 
 def _python_headers_impl(repository_ctx):
-  path = _get_config_var(repository_ctx, "INCLUDEPY")
+    path = _get_config_var(repository_ctx, "INCLUDEPY")
-  repository_ctx.symlink(path, "python")
+    repository_ctx.symlink(path, "python")
-  python3 = repository_ctx.which("python3")
+    python3 = repository_ctx.which("python3")
-  repository_ctx.file("BUILD.bazel", _build_file % python3)
+    repository_ctx.file("BUILD.bazel", _build_file % python3)
 
 # The system_python() repository rule exposes Python headers from the system.
 #

bazel/workspace_deps.bzl

@@ -23,10 +23,10 @@ def upb_deps():
             "rm python/google/protobuf/__init__.py",
             "rm python/google/protobuf/pyext/__init__.py",
             "rm python/google/protobuf/internal/__init__.py",
-        ]
+        ],
     )
 
-    rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53" # Latest @ 2021-06-23
+    rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53"  # Latest @ 2021-06-23
 
     maybe(
         http_archive,

python/BUILD

@@ -23,23 +23,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-load(
+load("//bazel:py_proto_library.bzl", "py_proto_library")
-    "//bazel:build_defs.bzl",
+load("//bazel:py_extension.bzl", "py_extension")  # copybara:strip_for_google3
-    "UPB_DEFAULT_COPTS",
+load("@rules_python//python:packaging.bzl", "py_wheel")
-)
-load(
-    "//bazel:py_proto_library.bzl",
-    "py_proto_library",
-)
-load(
-    "@rules_python//python:packaging.bzl",
-    "py_wheel",
-)
 
 licenses(["notice"])
 
-cc_binary(
+py_extension(
-    name = "message",
+    name = "_message",
     srcs = [
         "convert.c",
         "convert.h",
@@ -61,45 +52,38 @@ cc_binary(
         "repeated.c",
         "repeated.h",
     ],
-    copts = UPB_DEFAULT_COPTS + [
-        # The Python API requires patterns that are ISO C incompatible, like
-        # casts between function pointers and object pointers.
-        "-Wno-pedantic",
-    ],
-    # We use a linker script to hide all symbols except the entry point for
-    # the module.
-    linkopts = select({
-        "@platforms//os:linux": ["-Wl,--version-script,$(location :version_script.lds)"],
-        "@platforms//os:macos": [
-            "-Wl,-exported_symbol",
-            "-Wl,_PyInit__message",
-        ],
-    }),
-    linkshared = True,
-    linkstatic = True,
     deps = [
-        ":version_script.lds",
         "//:descriptor_upb_proto_reflection",
         "//:reflection",
+        "//:table",
         "//:textformat",
         "//:upb",
         "//upb/util:compare",
         "//upb/util:def_to_proto",
         "//upb/util:required_fields",
-        "@system_python//:python_headers",
     ],
 )
 
-cc_binary(
+py_extension(
-    name = "api_implementation",
+    name = "_api_implementation",
+    srcs = ["api_implementation.c"],
+)
+
+# copybara:strip_for_google3_begin
+
+py_test(
+    name = "minimal_test",
     srcs = [
-        "api_implementation.c",
+        "minimal_test.py",
+    ],
+    imports = ["."],
+    legacy_create_init = False,
+    deps = [
+        "//python:message_ext",
+        "@com_google_protobuf//:python_common_test_protos",
+        "@com_google_protobuf//:python_specific_test_protos",
+        "@com_google_protobuf//:python_srcs",
     ],
-    linkshared = True,
-    linkstatic = True,
-    # Enable once linker script is available.
-    #copts = ["-fvisibility=hidden"],
-    deps = ["@system_python//:python_headers"],
 )
 
 # Copy the extensions into the location recognized by Python.
@@ -108,14 +92,14 @@ EXT_SUFFIX = ".abi3.so"
 
 genrule(
     name = "copy_message",
-    srcs = [":message"],
+    srcs = [":_message"],
     outs = ["google/protobuf/pyext/_message" + EXT_SUFFIX],
     cmd = "cp $< $@",
 )
 
 genrule(
     name = "copy_api_implementation",
-    srcs = [":api_implementation"],
+    srcs = [":_api_implementation"],
     outs = ["google/protobuf/internal/_api_implementation" + EXT_SUFFIX],
     cmd = "cp $< $@",
     visibility = ["//python:__subpackages__"],
@@ -136,20 +120,6 @@ py_library(
     visibility = ["//python:__subpackages__"],
 )
 
-py_test(
-    name = "minimal_test",
-    srcs = [
-        "minimal_test.py",
-    ],
-    imports = ["."],
-    legacy_create_init = False,
-    deps = [
-        "//python:message_ext",
-        "@com_google_protobuf//:python_common_test_protos",
-        "@com_google_protobuf//:python_specific_test_protos",
-        "@com_google_protobuf//:python_srcs",
-    ],
-)
 
 py_proto_library(
     name = "well_known_proto_pb2",
@@ -189,3 +159,5 @@ py_wheel(
         "@com_google_protobuf//:python_srcs",
     ],
 )
+
+# copybara:strip_end

python/protobuf.h

@@ -193,7 +193,7 @@ PyObject* PyUpb_Forbidden_New(PyObject* cls, PyObject* args, PyObject* kwds);
 static inline void PyUpb_Dealloc(void* self) {
   PyTypeObject* tp = Py_TYPE(self);
   assert(PyType_GetFlags(tp) & Py_TPFLAGS_HEAPTYPE);
-  freefunc tp_free = PyType_GetSlot(tp, Py_tp_free);
+  freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free);
   tp_free(self);
   Py_DECREF(tp);
 }