From aaa338b285a82a11e0fd7e95722a3b790c89dc69 Mon Sep 17 00:00:00 2001 From: Marcel Hlopko Date: Fri, 17 Mar 2023 09:16:52 -0700 Subject: [PATCH] Configure build for the C++ backend PiperOrigin-RevId: 517429571 --- rust/BUILD | 37 ++- rust/aspects.bzl | 250 +++++++++++++++++ rust/cpp_kernel/BUILD | 18 ++ rust/cpp_kernel/cpp.rs | 51 ++++ rust/defs.bzl | 261 +++--------------- rust/lib.rs | 15 +- rust/test/BUILD | 18 +- .../rust_proto_library_unit_test/defs.bzl | 51 +++- .../rust_proto_library_unit_test.bzl | 57 +++- rust/{upb_backend => upb_kernel}/BUILD | 0 rust/{upb_backend => upb_kernel}/upb.rs | 23 +- rust/{upb_backend => upb_kernel}/upb_api.c | 0 src/google/protobuf/compiler/rust/BUILD.bazel | 1 + .../protobuf/compiler/rust/generator.cc | 112 +++++--- 14 files changed, 604 insertions(+), 290 deletions(-) create mode 100644 rust/aspects.bzl create mode 100644 rust/cpp_kernel/BUILD create mode 100644 rust/cpp_kernel/cpp.rs rename rust/{upb_backend => upb_kernel}/BUILD (100%) rename rust/{upb_backend => upb_kernel}/upb.rs (81%) rename rust/{upb_backend => upb_kernel}/upb_api.c (100%) diff --git a/rust/BUILD b/rust/BUILD index 30dbd1e4c1..230d1c6fde 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -1,6 +1,8 @@ # Protobuf Rust runtime packages. load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") +load("//third_party/bazel_skylib/rules:common_settings.bzl", "string_flag") + load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") package(default_visibility = ["//src/google/protobuf:__subpackages__"]) @@ -8,12 +10,23 @@ package(default_visibility = ["//src/google/protobuf:__subpackages__"]) rust_library( name = "protobuf", srcs = ["lib.rs"], - deps = ["//rust/upb_backend:upb"], + rustc_flags = select({ + ":use_upb_kernel": ["--cfg=upb_kernel"], + "//conditions:default": ["--cfg=cpp_kernel"], + }), + deps = select({ + ":use_upb_kernel": ["//rust/upb_kernel:upb"], + "//conditions:default": ["//rust/cpp_kernel:cpp"], + }), ) rust_test( name = "protobuf_test", crate = ":protobuf", + rustc_flags = select({ + ":use_upb_kernel": ["--cfg=upb_kernel"], + "//conditions:default": ["--cfg=cpp_kernel"], + }), tags = [ "not_build:arm", "notsan", @@ -23,8 +36,28 @@ rust_test( # TODO(b/270125787): Move to the right location once rust_proto_library is no longer experimental. proto_lang_toolchain( name = "proto_lang_toolchain", - command_line = "--rust_out=experimental-codegen=enabled:$(OUT)", + command_line = "--rust_out=experimental-codegen=enabled,kernel=" + select({ + ":use_upb_kernel": "upb", + "//conditions:default": "cpp", + }) + ":$(OUT)", progress_message = "Generating Rust proto_library %{label}", runtime = ":protobuf", visibility = ["//visibility:public"], ) + +# This flag controls what kernel all Rust Protobufs are using in the current build. +string_flag( + name = "rust_proto_library_kernel", + build_setting_default = "cpp", + values = [ + "upb", + "cpp", + ], +) + +config_setting( + name = "use_upb_kernel", + flag_values = { + ":rust_proto_library_kernel": "upb", + }, +) diff --git a/rust/aspects.bzl b/rust/aspects.bzl new file mode 100644 index 0000000000..6a8cb393ad --- /dev/null +++ b/rust/aspects.bzl @@ -0,0 +1,250 @@ +"""This file implements an experimental, do-not-use-kind of rust_proto_library. + +Disclaimer: This project is experimental, under heavy development, and should not +be used yet.""" + +# buildifier: disable=bzl-visibility +load("@rules_rust//rust/private:providers.bzl", "CrateInfo", "DepInfo", "DepVariantInfo") + +# buildifier: disable=bzl-visibility +load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action") +load("@rules_rust//rust:defs.bzl", "rust_common") +load("//third_party/upb/bazel:upb_proto_library.bzl", "UpbWrappedCcInfo", "upb_proto_library_aspect") + +proto_common = proto_common_do_not_use + +RustProtoInfo = provider( + doc = "Rust protobuf provider info", + fields = { + "dep_variant_info": "DepVariantInfo for the compiled Rust gencode (also covers its " + + "transitive dependencies)", + }, +) + +def _generate_rust_gencode( + ctx, + proto_info, + proto_lang_toolchain): + """Generates Rust gencode + + This function uses proto_common APIs and a ProtoLangToolchain to register an action + that invokes protoc with the right flags. + Args: + ctx (RuleContext): current rule context + proto_info (ProtoInfo): ProtoInfo of the proto_library target for which we are generating + gencode + proto_lang_toolchain (ProtoLangToolchainInfo): proto lang toolchain for Rust + Returns: + rs_outputs ([File]): generated Rust source files + """ + actions = ctx.actions + rs_outputs = proto_common.declare_generated_files( + actions = actions, + proto_info = proto_info, + extension = ".pb.rs", + ) + + proto_common.compile( + actions = ctx.actions, + proto_info = proto_info, + generated_files = rs_outputs, + proto_lang_toolchain_info = proto_lang_toolchain, + plugin_output = ctx.bin_dir.path, + ) + return rs_outputs + +def _get_crate_info(providers): + for provider in providers: + if hasattr(provider, "name"): + return provider + fail("Couldn't find a CrateInfo in the list of providers") + +def _get_dep_info(providers): + for provider in providers: + if hasattr(provider, "direct_crates"): + return provider + fail("Couldn't find a DepInfo in the list of providers") + +def _get_cc_info(providers): + for provider in providers: + if hasattr(provider, "linking_context"): + return provider + fail("Couldn't find a CcInfo in the list of providers") + +def _compile_rust(ctx, attr, src, extra_srcs, deps): + """Compiles a Rust source file. + + Eventually this function could be upstreamed into rules_rust and be made present in rust_common. + + Args: + ctx (RuleContext): The rule context. + attr (Attrs): The current rule's attributes (`ctx.attr` for rules, `ctx.rule.attr` for aspects) + src (File): The crate root source file to be compiled. + extra_srcs ([File]): Additional source files to include in the crate. + deps (List[DepVariantInfo]): A list of dependencies needed. + + Returns: + A DepVariantInfo provider. + """ + toolchain = ctx.toolchains["@rules_rust//rust:toolchain"] + output_hash = repr(hash(src.path)) + + # TODO(b/270124215): Use the import! macro once available + crate_name = ctx.label.name.replace("-", "_") + + lib_name = "{prefix}{name}-{lib_hash}{extension}".format( + prefix = "lib", + name = crate_name, + lib_hash = output_hash, + extension = ".rlib", + ) + + rmeta_name = "{prefix}{name}-{lib_hash}{extension}".format( + prefix = "lib", + name = crate_name, + lib_hash = output_hash, + extension = ".rmeta", + ) + + lib = ctx.actions.declare_file(lib_name) + rmeta = ctx.actions.declare_file(rmeta_name) + + providers = rustc_compile_action( + ctx = ctx, + attr = attr, + toolchain = toolchain, + crate_info = rust_common.create_crate_info( + name = crate_name, + type = "rlib", + root = src, + srcs = depset([src] + extra_srcs), + deps = depset(deps), + proc_macro_deps = depset([]), + aliases = {}, + output = lib, + metadata = rmeta, + edition = "2021", + is_test = False, + rustc_env = {}, + compile_data = depset([]), + compile_data_targets = depset([]), + owner = ctx.label, + ), + output_hash = output_hash, + ) + + return DepVariantInfo( + crate_info = _get_crate_info(providers), + dep_info = _get_dep_info(providers), + cc_info = _get_cc_info(providers), + build_info = None, + ) + +def _is_cc_proto_library(rule): + """Detects if the current rule is a cc_proto_library.""" + return rule.kind == "cc_proto_library" + +def _rust_upb_proto_aspect_impl(target, ctx): + """Implements the Rust protobuf aspect logic for UPB kernel.""" + return _rust_proto_aspect_common(target, ctx, is_upb = True) + +def _rust_cc_proto_aspect_impl(target, ctx): + """Implements the Rust protobuf aspect logic for C++ kernel.""" + return _rust_proto_aspect_common(target, ctx, is_upb = False) + +def _rust_proto_aspect_common(target, ctx, is_upb): + if RustProtoInfo in target: + return [] + + if _is_cc_proto_library(ctx.rule): + # This is cc_proto_library, but we need the RustProtoInfo provider of the proto_library that + # this aspect provides. Luckily this aspect has already been attached on the proto_library + # so we can just read the provider. + return [ctx.rule.attr.deps[0][RustProtoInfo]] + + proto_lang_toolchain = ctx.attr._proto_lang_toolchain[proto_common.ProtoLangToolchainInfo] + + gencode = _generate_rust_gencode(ctx, target[ProtoInfo], proto_lang_toolchain) + + runtime = proto_lang_toolchain.runtime + dep_variant_info_for_runtime = DepVariantInfo( + crate_info = runtime[CrateInfo] if CrateInfo in runtime else None, + dep_info = runtime[DepInfo] if DepInfo in runtime else None, + cc_info = runtime[CcInfo] if CcInfo in runtime else None, + build_info = None, + ) + dep_variant_info_for_native_gencode = DepVariantInfo( + cc_info = target[UpbWrappedCcInfo].cc_info_with_thunks if is_upb else target[CcInfo], + ) + + proto_dep = getattr(ctx.rule.attr, "deps", []) + dep_variant_info = _compile_rust( + ctx = ctx, + attr = ctx.rule.attr, + src = gencode[0], + extra_srcs = gencode[1:], + deps = [dep_variant_info_for_runtime, dep_variant_info_for_native_gencode] + ( + [proto_dep[0][RustProtoInfo].dep_variant_info] if proto_dep else [] + ), + ) + return [RustProtoInfo(dep_variant_info = dep_variant_info)] + +def _make_proto_library_aspect(is_upb): + return aspect( + implementation = (_rust_upb_proto_aspect_impl if is_upb else _rust_cc_proto_aspect_impl), + attr_aspects = ["deps"], + # Since we can reference upb_proto_library_aspect by name, we can just ask Bazel to attach + # it here. + requires = ([upb_proto_library_aspect] if is_upb else []), + required_aspect_providers = ([] if is_upb else [CcInfo]), + attrs = { + "_cc_toolchain": attr.label( + doc = ( + "In order to use find_cc_toolchain, your rule has to depend " + + "on C++ toolchain. See `@rules_cc//cc:find_cc_toolchain.bzl` " + + "docs for details." + ), + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), + "_collect_cc_coverage": attr.label( + default = Label("@rules_rust//util:collect_coverage"), + executable = True, + cfg = "exec", + ), + "_error_format": attr.label( + default = Label("@rules_rust//:error_format"), + ), + "_extra_exec_rustc_flag": attr.label( + default = Label("@rules_rust//:extra_exec_rustc_flag"), + ), + "_extra_exec_rustc_flags": attr.label( + default = Label("@rules_rust//:extra_exec_rustc_flags"), + ), + "_extra_rustc_flag": attr.label( + default = Label("@rules_rust//:extra_rustc_flag"), + ), + "_extra_rustc_flags": attr.label( + default = Label("@rules_rust//:extra_rustc_flags"), + ), + "_process_wrapper": attr.label( + doc = "A process wrapper for running rustc on all platforms.", + default = Label("@rules_rust//util/process_wrapper"), + executable = True, + allow_single_file = True, + cfg = "exec", + ), + "_proto_lang_toolchain": attr.label( + default = Label("//rust:proto_lang_toolchain"), + ), + }, + fragments = ["cpp"], + host_fragments = ["cpp"], + toolchains = [ + str(Label("@rules_rust//rust:toolchain")), + "@bazel_tools//tools/cpp:toolchain_type", + ], + incompatible_use_toolchain_transition = True, + ) + +rust_upb_proto_library_aspect = _make_proto_library_aspect(is_upb = True) +rust_cc_proto_library_aspect = _make_proto_library_aspect(is_upb = False) diff --git a/rust/cpp_kernel/BUILD b/rust/cpp_kernel/BUILD new file mode 100644 index 0000000000..415bd96f50 --- /dev/null +++ b/rust/cpp_kernel/BUILD @@ -0,0 +1,18 @@ +# This package contains Rust protobuf runtime implementation built on top of the C++ backend. + +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "cpp", + srcs = ["cpp.rs"], + visibility = ["//src/google/protobuf:__subpackages__"], +) + +rust_test( + name = "cpp_test", + crate = ":cpp", + tags = [ + "not_build:arm", + "notsan", + ], +) diff --git a/rust/cpp_kernel/cpp.rs b/rust/cpp_kernel/cpp.rs new file mode 100644 index 0000000000..2451be5f72 --- /dev/null +++ b/rust/cpp_kernel/cpp.rs @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Rust Protobuf runtime using the C++ kernel. + +use std::boxed::Box; + +/// TODO(b/272728844): Replace this placeholder code with a real implementation. +#[repr(C)] +pub struct Arena { + _data: [u8; 0], +} + +impl Arena { + pub unsafe fn new() -> *mut Self { + let arena = Box::new(Arena { _data: [] }); + Box::leak(arena) as *mut _ + } + + pub unsafe fn free(arena: *mut Self) { + let arena = Box::from_raw(arena); + std::mem::drop(arena); + } +} diff --git a/rust/defs.bzl b/rust/defs.bzl index af42f51fdf..eaefe8a746 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -3,228 +3,41 @@ Disclaimer: This project is experimental, under heavy development, and should not be used yet.""" -# buildifier: disable=bzl-visibility -load("@rules_rust//rust/private:providers.bzl", "CrateInfo", "DepInfo", "DepVariantInfo") - -# buildifier: disable=bzl-visibility -load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action") -load("@rules_rust//rust:defs.bzl", "rust_common") -load("//third_party/upb/bazel:upb_proto_library.bzl", "UpbWrappedCcInfo", "upb_proto_library_aspect") - -proto_common = proto_common_do_not_use - -RustProtoInfo = provider( - doc = "Rust protobuf provider info", - fields = { - "dep_variant_info": "DepVariantInfo for the compiled Rust gencode (also covers its " + - "transitive dependencies)", - }, +load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library") +load( + "//rust:aspects.bzl", + "RustProtoInfo", + "rust_cc_proto_library_aspect", + "rust_upb_proto_library_aspect", ) -def _generate_rust_gencode( - ctx, - proto_info, - proto_lang_toolchain): - """Generates Rust gencode - - This function uses proto_common APIs and a ProtoLangToolchain to register an action - that invokes protoc with the right flags. - Args: - ctx (RuleContext): current rule context - proto_info (ProtoInfo): ProtoInfo of the proto_library target for which we are generating - gencode - proto_lang_toolchain (ProtoLangToolchainInfo): proto lang toolchain for Rust - Returns: - rs_outputs ([File]): generated Rust source files - """ - actions = ctx.actions - rs_outputs = proto_common.declare_generated_files( - actions = actions, - proto_info = proto_info, - extension = ".pb.rs", - ) - - proto_common.compile( - actions = ctx.actions, - proto_info = proto_info, - generated_files = rs_outputs, - proto_lang_toolchain_info = proto_lang_toolchain, - plugin_output = ctx.bin_dir.path, - ) - return rs_outputs - -def _get_crate_info(providers): - for provider in providers: - if hasattr(provider, "name"): - return provider - fail("Couldn't find a CrateInfo in the list of providers") - -def _get_dep_info(providers): - for provider in providers: - if hasattr(provider, "direct_crates"): - return provider - fail("Couldn't find a DepInfo in the list of providers") - -def _get_cc_info(providers): - for provider in providers: - if hasattr(provider, "linking_context"): - return provider - fail("Couldn't find a CcInfo in the list of providers") +def rust_proto_library(name, deps, **args): + """Declares all the boilerplate needed to use Rust protobufs conveniently. -def _compile_rust(ctx, attr, src, extra_srcs, deps): - """Compiles a Rust source file. + Hopefully no user will ever need to read this code. - Eventually this function could be upstreamed into rules_rust and be made present in rust_common. Args: - ctx (RuleContext): The rule context. - attr (Attrs): The current rule's attributes (`ctx.attr` for rules, `ctx.rule.attr` for aspects) - src (File): The crate root source file to be compiled. - extra_srcs ([File]): Additional source files to include in the crate. - deps (List[DepVariantInfo]): A list of dependencies needed. - - Returns: - A DepVariantInfo provider. + name: name of the Rust protobuf target. + deps: proto_library target for which to generate Rust gencode. + **args: other args passed to the rust__proto_library targets. """ - toolchain = ctx.toolchains["@rules_rust//rust:toolchain"] - output_hash = repr(hash(src.path)) - - # TODO(b/270124215): Use the import! macro once available - crate_name = ctx.label.name.replace("-", "_") - - lib_name = "{prefix}{name}-{lib_hash}{extension}".format( - prefix = "lib", - name = crate_name, - lib_hash = output_hash, - extension = ".rlib", - ) - - rmeta_name = "{prefix}{name}-{lib_hash}{extension}".format( - prefix = "lib", - name = crate_name, - lib_hash = output_hash, - extension = ".rmeta", + if not name.endswith("_rust_proto"): + fail("Name of each rust_proto_library target should end with `_rust_proto`") + native.alias( + name = name, + actual = select({ + "//rust:use_upb_kernel": name + "_upb_kernel", + "//conditions:default": name + "_cpp_kernel", + }), ) - lib = ctx.actions.declare_file(lib_name) - rmeta = ctx.actions.declare_file(rmeta_name) + rust_upb_proto_library(name = name + "_upb_kernel", deps = deps, **args) - providers = rustc_compile_action( - ctx = ctx, - attr = attr, - toolchain = toolchain, - crate_info = rust_common.create_crate_info( - name = crate_name, - type = "rlib", - root = src, - srcs = depset([src] + extra_srcs), - deps = depset(deps), - proc_macro_deps = depset([]), - aliases = {}, - output = lib, - metadata = rmeta, - edition = "2021", - is_test = False, - rustc_env = {}, - compile_data = depset([]), - compile_data_targets = depset([]), - owner = ctx.label, - ), - output_hash = output_hash, - ) - - return DepVariantInfo( - crate_info = _get_crate_info(providers), - dep_info = _get_dep_info(providers), - cc_info = _get_cc_info(providers), - build_info = None, - ) - -def _rust_proto_aspect_impl(target, ctx): - if ProtoInfo not in target: - return None - - proto_lang_toolchain = ctx.attr._proto_lang_toolchain[proto_common.ProtoLangToolchainInfo] - - gencode = _generate_rust_gencode(ctx, target[ProtoInfo], proto_lang_toolchain) - - runtime = proto_lang_toolchain.runtime - dep_variant_info_for_runtime = DepVariantInfo( - crate_info = runtime[CrateInfo] if CrateInfo in runtime else None, - dep_info = runtime[DepInfo] if DepInfo in runtime else None, - cc_info = runtime[CcInfo] if CcInfo in runtime else None, - build_info = None, - ) - - upb_gencode_cc_info = target[UpbWrappedCcInfo].cc_info_with_thunks - upb_gencode_dep_variant_info = DepVariantInfo(cc_info = upb_gencode_cc_info) - - proto_dep = getattr(ctx.rule.attr, "deps", []) - dep_variant_info = _compile_rust( - ctx = ctx, - attr = ctx.rule.attr, - src = gencode[0], - extra_srcs = gencode[1:], - deps = [dep_variant_info_for_runtime, upb_gencode_dep_variant_info] + ( - [proto_dep[0][RustProtoInfo].dep_variant_info] if proto_dep else [] - ), - ) - return [RustProtoInfo( - dep_variant_info = dep_variant_info, - )] - -rust_proto_library_aspect = aspect( - implementation = _rust_proto_aspect_impl, - attr_aspects = ["deps"], - requires = [upb_proto_library_aspect], - attrs = { - "_cc_toolchain": attr.label( - doc = ( - "In order to use find_cc_toolchain, your rule has to depend " + - "on C++ toolchain. See `@rules_cc//cc:find_cc_toolchain.bzl` " + - "docs for details." - ), - default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), - ), - "_collect_cc_coverage": attr.label( - default = Label("@rules_rust//util:collect_coverage"), - executable = True, - cfg = "exec", - ), - "_error_format": attr.label( - default = Label("@rules_rust//:error_format"), - ), - "_extra_exec_rustc_flag": attr.label( - default = Label("@rules_rust//:extra_exec_rustc_flag"), - ), - "_extra_exec_rustc_flags": attr.label( - default = Label("@rules_rust//:extra_exec_rustc_flags"), - ), - "_extra_rustc_flag": attr.label( - default = Label("@rules_rust//:extra_rustc_flag"), - ), - "_extra_rustc_flags": attr.label( - default = Label("@rules_rust//:extra_rustc_flags"), - ), - "_process_wrapper": attr.label( - doc = "A process wrapper for running rustc on all platforms.", - default = Label("@rules_rust//util/process_wrapper"), - executable = True, - allow_single_file = True, - cfg = "exec", - ), - "_proto_lang_toolchain": attr.label( - default = Label("//rust:proto_lang_toolchain"), - ), - }, - fragments = ["cpp"], - host_fragments = ["cpp"], - toolchains = [ - str(Label("@rules_rust//rust:toolchain")), - "@bazel_tools//tools/cpp:toolchain_type", - ], - incompatible_use_toolchain_transition = True, -) + _cc_proto_name = name.removesuffix("_rust_proto") + "_cc_proto" + if not native.existing_rule(_cc_proto_name): + cc_proto_library(name = _cc_proto_name, deps = deps, **args) + rust_cc_proto_library(name = name + "_cpp_kernel", deps = [_cc_proto_name], **args) def _rust_proto_library_impl(ctx): deps = ctx.attr.deps @@ -238,13 +51,17 @@ def _rust_proto_library_impl(ctx): dep_variant_info = rust_proto_info.dep_variant_info return [dep_variant_info.crate_info, dep_variant_info.dep_info, dep_variant_info.cc_info] -rust_proto_library = rule( - implementation = _rust_proto_library_impl, - attrs = { - "deps": attr.label_list( - mandatory = True, - providers = [ProtoInfo], - aspects = [rust_proto_library_aspect], - ), - }, -) +def _make_rust_proto_library(is_upb): + return rule( + implementation = _rust_proto_library_impl, + attrs = { + "deps": attr.label_list( + mandatory = True, + providers = [ProtoInfo] if is_upb else [CcInfo], + aspects = [rust_upb_proto_library_aspect if is_upb else rust_cc_proto_library_aspect], + ), + }, + ) + +rust_upb_proto_library = _make_rust_proto_library(is_upb = True) +rust_cc_proto_library = _make_rust_proto_library(is_upb = False) diff --git a/rust/lib.rs b/rust/lib.rs index 0aaeb2d97b..ca2f7f0fa0 100644 --- a/rust/lib.rs +++ b/rust/lib.rs @@ -30,7 +30,12 @@ //! Rust Protobuf Runtime -pub use upb::*; +#[cfg(cpp_kernel)] +pub extern crate cpp as __runtime; +#[cfg(upb_kernel)] +pub extern crate upb as __runtime; + +pub use __runtime::Arena; use std::ops::Deref; use std::ptr::NonNull; @@ -41,11 +46,11 @@ use std::slice; pub struct SerializedData { data: NonNull, len: usize, - arena: *mut upb_Arena, + arena: *mut Arena, } impl SerializedData { - pub unsafe fn from_raw_parts(arena: *mut upb_Arena, data: NonNull, len: usize) -> Self { + pub unsafe fn from_raw_parts(arena: *mut Arena, data: NonNull, len: usize) -> Self { SerializedData { arena, data, len } } } @@ -59,7 +64,7 @@ impl Deref for SerializedData { impl Drop for SerializedData { fn drop(&mut self) { - unsafe { upb_Arena_Free(self.arena) }; + unsafe { Arena::free(self.arena) }; } } @@ -69,7 +74,7 @@ mod tests { #[test] fn test_serialized_data_roundtrip() { - let arena = unsafe { upb_Arena_New() }; + let arena = unsafe { Arena::new() }; let original_data = b"Hello world"; let len = original_data.len(); diff --git a/rust/test/BUILD b/rust/test/BUILD index 780d3c67ed..9298b3bc32 100644 --- a/rust/test/BUILD +++ b/rust/test/BUILD @@ -2,7 +2,7 @@ load("//rust:defs.bzl", "rust_proto_library") load("@rules_rust//rust:defs.bzl", "rust_test") rust_proto_library( - name = "unittest_rs_proto", + name = "unittest_rust_proto", testonly = True, deps = ["//third_party/protobuf:unittest_proto"], ) @@ -13,9 +13,12 @@ rust_test( # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. tags = [ "not_build:arm", + # TODO(b/225892643): Enable once we use Blaze-bootstrapped Rust toolchain. "notsan", + # TODO(b/243126140): Enable msan once we support sanitizers with Rust. + "nomsan", ], - deps = [":unittest_rs_proto"], + deps = [":unittest_rust_proto"], ) proto_library( @@ -31,12 +34,12 @@ proto_library( ) rust_proto_library( - name = "parent_rs_proto", + name = "parent_rust_proto", deps = [":parent_proto"], ) rust_proto_library( - name = "child_rs_proto", + name = "child_rust_proto", deps = [":child_proto"], ) @@ -46,10 +49,13 @@ rust_test( # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. tags = [ "not_build:arm", + # TODO(b/225892643): Enable once we use Blaze-bootstrapped Rust toolchain. "notsan", + # TODO(b/243126140): Enable msan once we support sanitizers with Rust. + "nomsan", ], deps = [ - ":child_rs_proto", - ":parent_rs_proto", + ":child_rust_proto", + ":parent_rust_proto", ], ) diff --git a/rust/test/rust_proto_library_unit_test/defs.bzl b/rust/test/rust_proto_library_unit_test/defs.bzl index 81741aad62..9c4c1ec838 100644 --- a/rust/test/rust_proto_library_unit_test/defs.bzl +++ b/rust/test/rust_proto_library_unit_test/defs.bzl @@ -1,19 +1,58 @@ """Support for rust_proto_library_aspect unit-tests.""" -load("//rust:defs.bzl", "RustProtoInfo", "rust_proto_library_aspect") +load( + "//rust:aspects.bzl", + "RustProtoInfo", + "rust_cc_proto_library_aspect", + "rust_upb_proto_library_aspect", +) ActionsInfo = provider( - doc = ("A provider that exposes what actions were registered by rust_proto_library_aspect " + + doc = ("A provider that exposes what actions were registered by rust_proto_library aspects " + "on proto_libraries."), fields = {"actions": "List[Action]: actions registered on proto_libraries."}, ) -def _attach_aspect_impl(ctx): +def _attach_upb_aspect_impl(ctx): return [ctx.attr.dep[RustProtoInfo], ActionsInfo(actions = ctx.attr.dep.actions)] -attach_aspect = rule( - implementation = _attach_aspect_impl, +attach_upb_aspect = rule( + implementation = _attach_upb_aspect_impl, + attrs = { + "dep": attr.label(aspects = [rust_upb_proto_library_aspect]), + }, +) + +CcAspectHelperInfo = provider( + fields = { + "rust_proto_info": "RustProtoInfo from the proto_library", + "actions_info": "Actions of the proto_library", + }, + doc = "A provider passing data from proto_library through cc_proto_library", +) + +def _cc_aspect_helper_impl(_target, ctx): + if ctx.rule.kind == "cc_proto_library": + return CcAspectHelperInfo( + rust_proto_info = ctx.rule.attr.deps[0][RustProtoInfo], + actions_info = ActionsInfo(actions = ctx.rule.attr.deps[0].actions), + ) + + return [] + +_cc_aspect_helper = aspect( + implementation = _cc_aspect_helper_impl, + requires = [rust_cc_proto_library_aspect], + attr_aspects = ["deps"], +) + +def _attach_cc_aspect_impl(ctx): + helper = ctx.attr.dep[CcAspectHelperInfo] + return [helper.rust_proto_info, helper.actions_info] + +attach_cc_aspect = rule( + implementation = _attach_cc_aspect_impl, attrs = { - "dep": attr.label(aspects = [rust_proto_library_aspect]), + "dep": attr.label(aspects = [_cc_aspect_helper]), }, ) diff --git a/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl b/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl index 5379207e1c..06386615c2 100644 --- a/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl +++ b/rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl @@ -1,8 +1,9 @@ """This module contains unit tests for rust_proto_library and its aspect.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load(":defs.bzl", "ActionsInfo", "attach_aspect") -load("//rust:defs.bzl", "RustProtoInfo") +load(":defs.bzl", "ActionsInfo", "attach_cc_aspect", "attach_upb_aspect") +load("//rust:aspects.bzl", "RustProtoInfo") +load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library") def _find_action_with_mnemonic(actions, mnemonic): action = [a for a in actions if a.mnemonic == mnemonic] @@ -49,7 +50,7 @@ def _find_linker_input(rust_proto_info, basename_substring): #################################################################################################### -def _rust_aspect_test_impl(ctx): +def _rust_upb_aspect_test_impl(ctx): env = analysistest.begin(ctx) target_under_test = analysistest.target_under_test(env) actions = target_under_test[ActionsInfo].actions @@ -67,14 +68,45 @@ def _rust_aspect_test_impl(ctx): return analysistest.end(env) -rust_aspect_test = analysistest.make(_rust_aspect_test_impl) +rust_upb_aspect_test = analysistest.make(_rust_upb_aspect_test_impl) -def _test_aspect(): - attach_aspect(name = "child_proto_with_aspect", dep = ":child_proto") +def _test_upb_aspect(): + attach_upb_aspect(name = "child_proto_with_upb_aspect", dep = ":child_proto") - rust_aspect_test( - name = "rust_aspect_test", - target_under_test = ":child_proto_with_aspect", + rust_upb_aspect_test( + name = "rust_upb_aspect_test", + target_under_test = ":child_proto_with_upb_aspect", + # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. + tags = ["not_build:arm"], + ) + +#################################################################################################### +def _rust_cc_aspect_test_impl(ctx): + env = analysistest.begin(ctx) + target_under_test = analysistest.target_under_test(env) + actions = target_under_test[ActionsInfo].actions + rustc_action = _find_action_with_mnemonic(actions, "Rustc") + + # The action needs to have the Rust runtime as an input + _find_rust_lib_input(rustc_action.inputs, "protobuf") + + # The action needs to produce a .rlib artifact (sometimes .rmeta as well, not tested here). + asserts.true(env, rustc_action.outputs.to_list()[0].path.endswith(".rlib")) + + # The aspect needs to provide CcInfo that passes UPB gencode to the eventual linking. + _find_linker_input(target_under_test[RustProtoInfo], "child.pb") + _find_linker_input(target_under_test[RustProtoInfo], "parent.pb") + + return analysistest.end(env) + +rust_cc_aspect_test = analysistest.make(_rust_cc_aspect_test_impl) + +def _test_cc_aspect(): + attach_cc_aspect(name = "child_proto_with_cc_aspect", dep = ":child_cc_proto") + + rust_cc_aspect_test( + name = "rust_cc_aspect_test", + target_under_test = ":child_proto_with_cc_aspect", # TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain. tags = ["not_build:arm"], ) @@ -88,12 +120,15 @@ def rust_proto_library_unit_test(name): name: name of the test suite""" native.proto_library(name = "parent_proto", srcs = ["parent.proto"]) native.proto_library(name = "child_proto", srcs = ["child.proto"], deps = [":parent_proto"]) + cc_proto_library(name = "child_cc_proto", deps = [":child_proto"]) - _test_aspect() + _test_upb_aspect() + _test_cc_aspect() native.test_suite( name = name, tests = [ - ":rust_aspect_test", + ":rust_upb_aspect_test", + ":rust_cc_aspect_test", ], ) diff --git a/rust/upb_backend/BUILD b/rust/upb_kernel/BUILD similarity index 100% rename from rust/upb_backend/BUILD rename to rust/upb_kernel/BUILD diff --git a/rust/upb_backend/upb.rs b/rust/upb_kernel/upb.rs similarity index 81% rename from rust/upb_backend/upb.rs rename to rust/upb_kernel/upb.rs index 35a09fa86f..f07089f197 100644 --- a/rust/upb_backend/upb.rs +++ b/rust/upb_kernel/upb.rs @@ -28,17 +28,28 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Rust bindings for UPB +// Rust Protobuf runtime using the UPB kernel. +/// Represents UPB's upb_Arena. #[repr(C)] -pub struct upb_Arena { +pub struct Arena { _data: [u8; 0], _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } +impl Arena { + pub unsafe fn new() -> *mut Self { + upb_Arena_New() + } + + pub unsafe fn free(arena: *mut Self) { + upb_Arena_Free(arena) + } +} + extern "C" { - pub fn upb_Arena_New() -> *mut upb_Arena; - pub fn upb_Arena_Free(arena: *mut upb_Arena); + pub fn upb_Arena_New() -> *mut Arena; + pub fn upb_Arena_Free(arena: *mut Arena); } #[cfg(test)] @@ -47,7 +58,7 @@ mod tests { #[test] fn test_arena_new_and_free() { - let arena = unsafe { upb_Arena_New() }; - unsafe { upb_Arena_Free(arena) }; + let arena = unsafe { Arena::new() }; + unsafe { Arena::free(arena) }; } } diff --git a/rust/upb_backend/upb_api.c b/rust/upb_kernel/upb_api.c similarity index 100% rename from rust/upb_backend/upb_api.c rename to rust/upb_kernel/upb_api.c diff --git a/src/google/protobuf/compiler/rust/BUILD.bazel b/src/google/protobuf/compiler/rust/BUILD.bazel index faddffc6ed..7a75c61bab 100644 --- a/src/google/protobuf/compiler/rust/BUILD.bazel +++ b/src/google/protobuf/compiler/rust/BUILD.bazel @@ -19,5 +19,6 @@ cc_library( "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/types:optional", ], ) diff --git a/src/google/protobuf/compiler/rust/generator.cc b/src/google/protobuf/compiler/rust/generator.cc index 171a2fb4dd..e5c81adc51 100644 --- a/src/google/protobuf/compiler/rust/generator.cc +++ b/src/google/protobuf/compiler/rust/generator.cc @@ -38,6 +38,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/io/printer.h" @@ -57,6 +58,27 @@ bool ExperimentalRustGeneratorEnabled( }); } +// Marks which kernel Rust codegen assumes and generates gencode for. +enum class Kernel { + kUpb, + kCpp, +}; + +absl::optional ParsekernelConfiguration( + const std::vector>& options) { + for (const auto& pair : options) { + if (pair.first == "kernel") { + if (pair.second == "upb") { + return Kernel::kUpb; + } + if (pair.second == "cpp") { + return Kernel::kCpp; + } + } + } + return absl::nullopt; +} + std::string get_crate_name(const FileDescriptor* dependency) { absl::string_view path = dependency->name(); auto basename = path.substr(path.rfind('/') + 1); @@ -81,17 +103,23 @@ bool RustGenerator::Generate(const FileDescriptor* file, return false; } + absl::optional kernel = ParsekernelConfiguration(options); + if (!kernel.has_value()) { + *error = + "Mandatory option `kernel` missing, please specify `cpp` or " + "`upb`."; + return false; + } + auto basename = StripProto(file->name()); auto outfile = absl::WrapUnique( generator_context->Open(absl::StrCat(basename, ".pb.rs"))); google::protobuf::io::Printer p(outfile.get()); - // TODO(b/270138878): Remove `do_nothing` import once we have real logic. This - // is there only to smoke test rustc actions in rust_proto_library. p.Emit(R"rs( extern crate protobuf as __pb; extern crate std as __std; - + )rs"); // TODO(b/270124215): Delete the following "placeholder impl" of `import @@ -101,7 +129,7 @@ bool RustGenerator::Generate(const FileDescriptor* file, const FileDescriptor* dep = file->public_dependency(i); std::string crate_name = get_crate_name(dep); for (int j = 0; j < dep->message_type_count(); ++j) { - // TODO(b/270138878): Implement real logic + // TODO(b/272728844): Implement real logic p.Emit( {{"crate", crate_name}, {"type_name", dep->message_type(j)->name()}}, R"rs( @@ -111,38 +139,58 @@ bool RustGenerator::Generate(const FileDescriptor* file, } for (int i = 0; i < file->message_type_count(); ++i) { - // TODO(b/270138878): Implement real logic + // TODO(b/272728844): Implement real logic std::string full_name = file->message_type(i)->full_name(); absl::StrReplaceAll({{".", "_"}}, &full_name); - p.Emit({{"Msg", file->message_type(i)->name()}, {"pkg_Msg", full_name}}, - R"rs( - pub struct $Msg$ { - msg: ::__std::ptr::NonNull, - arena: *mut ::__pb::upb_Arena, - } - - impl $Msg$ { - pub fn new() -> $Msg$ { - let arena = unsafe { ::__pb::upb_Arena_New() }; - let msg = unsafe { $pkg_Msg$_new(arena) }; - $Msg$ { msg, arena } - } - pub fn serialize(&self) -> ::__pb::SerializedData { - let arena = unsafe { ::__pb::upb_Arena_New() }; - let mut len = 0; - let chars = unsafe { $pkg_Msg$_serialize(self.msg, arena, &mut len) }; - unsafe {::__pb::SerializedData::from_raw_parts(arena, chars, len)} - } + switch (*kernel) { + case Kernel::kUpb: { + p.Emit({{"Msg", file->message_type(i)->name()}, {"pkg_Msg", full_name}}, + R"rs( + pub struct $Msg$ { + msg: ::__std::ptr::NonNull, + arena: *mut ::__pb::Arena, + } + + impl $Msg$ { + pub fn new() -> Self { + let arena = unsafe { ::__pb::Arena::new() }; + let msg = unsafe { $pkg_Msg$_new(arena) }; + $Msg$ { msg, arena } + } + pub fn serialize(&self) -> ::__pb::SerializedData { + let arena = unsafe { ::__pb::__runtime::upb_Arena_New() }; + let mut len = 0; + let chars = unsafe { $pkg_Msg$_serialize(self.msg, arena, &mut len) }; + unsafe {::__pb::SerializedData::from_raw_parts(arena, chars, len)} + } + } + + extern "C" { + fn $pkg_Msg$_new(arena: *mut ::__pb::Arena) -> ::__std::ptr::NonNull; + fn $pkg_Msg$_serialize( + msg: ::__std::ptr::NonNull, + arena: *mut ::__pb::Arena, + len: &mut usize) -> ::__std::ptr::NonNull; + } + )rs"); + break; } - - extern "C" { - fn $pkg_Msg$_new(arena: *mut ::__pb::upb_Arena) -> ::__std::ptr::NonNull; - fn $pkg_Msg$_serialize( - msg: ::__std::ptr::NonNull, - arena: *mut ::__pb::upb_Arena, - len: &mut usize) -> ::__std::ptr::NonNull; + case Kernel::kCpp: { + // TODO(b/272728844): Implement real logic + p.Emit({{"Msg", file->message_type(i)->name()}}, + R"rs( + pub struct $Msg$ { + msg: ::__std::ptr::NonNull, + } + + impl $Msg$ { + pub fn new() -> Self { Self { msg: ::__std::ptr::NonNull::dangling() }} + pub fn serialize(&self) -> Vec { vec![] } + } + )rs"); + break; } - )rs"); + } } return true;