Configure the build for the Rust UPB backend

In this CL we're adding the barebones infrastructure to generate Rust proto messages using UPB as a backend. The API is what we call a V0, not yet production-quality, not yet rigorously designed, just something to enable parallel work.

The interesting part of switching backend between UPB and C++ will come in a followup.

PiperOrigin-RevId: 517089760
pull/12237/head
Marcel Hlopko 2 years ago committed by Copybara-Service
parent fe758fa12f
commit ab9f1ab58a
  1. 12
      rust/BUILD
  2. 9
      rust/defs.bzl
  3. 55
      rust/lib.rs
  4. 10
      rust/test/BUILD
  5. 19
      rust/test/child_parent_test.rs
  6. 33
      rust/test/rust_proto_library_unit_test/empty.cc
  7. 33
      rust/test/rust_proto_library_unit_test/empty.rs
  8. 82
      rust/test/rust_proto_library_unit_test/rust_proto_library_unit_test.bzl
  9. 8
      rust/test/unittest_proto_test.rs
  10. 27
      rust/upb_backend/BUILD
  11. 53
      rust/upb_backend/upb.rs
  12. 30
      rust/upb_backend/upb_api.c
  13. 47
      src/google/protobuf/compiler/rust/generator.cc

@ -1,6 +1,6 @@
# Protobuf Rust runtime packages.
load("@rules_rust//rust:defs.bzl", "rust_library")
load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain")
package(default_visibility = ["//src/google/protobuf:__subpackages__"])
@ -8,6 +8,16 @@ package(default_visibility = ["//src/google/protobuf:__subpackages__"])
rust_library(
name = "protobuf",
srcs = ["lib.rs"],
deps = ["//rust/upb_backend:upb"],
)
rust_test(
name = "protobuf_test",
crate = ":protobuf",
tags = [
"not_build:arm",
"notsan",
],
)
# TODO(b/270125787): Move to the right location once rust_proto_library is no longer experimental.

@ -9,6 +9,7 @@ load("@rules_rust//rust/private:providers.bzl", "CrateInfo", "DepInfo", "DepVari
# buildifier: disable=bzl-visibility
load("@rules_rust//rust/private:rustc.bzl", "rustc_compile_action")
load("@rules_rust//rust:defs.bzl", "rust_common")
load("//third_party/upb/bazel:upb_proto_library.bzl", "UpbWrappedCcInfo", "upb_proto_library_aspect")
proto_common = proto_common_do_not_use
@ -155,13 +156,18 @@ def _rust_proto_aspect_impl(target, ctx):
build_info = None,
)
upb_gencode_cc_info = target[UpbWrappedCcInfo].cc_info_with_thunks
upb_gencode_dep_variant_info = DepVariantInfo(cc_info = upb_gencode_cc_info)
proto_dep = getattr(ctx.rule.attr, "deps", [])
dep_variant_info = _compile_rust(
ctx = ctx,
attr = ctx.rule.attr,
src = gencode[0],
extra_srcs = gencode[1:],
deps = [dep_variant_info_for_runtime] + ([proto_dep[0][RustProtoInfo].dep_variant_info] if proto_dep else []),
deps = [dep_variant_info_for_runtime, upb_gencode_dep_variant_info] + (
[proto_dep[0][RustProtoInfo].dep_variant_info] if proto_dep else []
),
)
return [RustProtoInfo(
dep_variant_info = dep_variant_info,
@ -170,6 +176,7 @@ def _rust_proto_aspect_impl(target, ctx):
rust_proto_library_aspect = aspect(
implementation = _rust_proto_aspect_impl,
attr_aspects = ["deps"],
requires = [upb_proto_library_aspect],
attrs = {
"_cc_toolchain": attr.label(
doc = (

@ -30,7 +30,56 @@
//! Rust Protobuf Runtime
// Not yet implemented.
pub use upb::*;
// TODO(b/270138878): Remove once we have real logic in the runtime.
pub fn do_nothing() {}
use std::ops::Deref;
use std::ptr::NonNull;
use std::slice;
/// Represents serialized Protobuf wire format data. It's typically produced by
/// `<Message>.serialize()`.
pub struct SerializedData {
data: NonNull<u8>,
len: usize,
arena: *mut upb_Arena,
}
impl SerializedData {
pub unsafe fn from_raw_parts(arena: *mut upb_Arena, data: NonNull<u8>, len: usize) -> Self {
SerializedData { arena, data, len }
}
}
impl Deref for SerializedData {
type Target = [u8];
fn deref(&self) -> &Self::Target {
unsafe { slice::from_raw_parts(self.data.as_ptr() as *const _, self.len) }
}
}
impl Drop for SerializedData {
fn drop(&mut self) {
unsafe { upb_Arena_Free(self.arena) };
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_serialized_data_roundtrip() {
let arena = unsafe { upb_Arena_New() };
let original_data = b"Hello world";
let len = original_data.len();
let serialized_data = unsafe {
SerializedData::from_raw_parts(
arena,
NonNull::new(original_data as *const _ as *mut _).unwrap(),
len,
)
};
assert_eq!(&*serialized_data, b"Hello world");
}
}

@ -11,7 +11,10 @@ rust_test(
name = "unittest_proto_test",
srcs = ["unittest_proto_test.rs"],
# TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain.
tags = ["not_build:arm"],
tags = [
"not_build:arm",
"notsan",
],
deps = [":unittest_rs_proto"],
)
@ -41,7 +44,10 @@ rust_test(
name = "child_parent_test",
srcs = ["child_parent_test.rs"],
# TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain.
tags = ["not_build:arm"],
tags = [
"not_build:arm",
"notsan",
],
deps = [
":child_rs_proto",
":parent_rs_proto",

@ -28,10 +28,21 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
fn main() {
let _child = child_proto::Child {};
let _parent = parent_proto::Parent {};
#[test]
fn test_canonical_types() {
let _child = child_proto::Child::new();
let _parent = parent_proto::Parent::new();
// Parent from child_proto crate should be the same type as Parent from
// parent_proto crate.
let _parent_from_child: child_proto::Parent = parent_proto::Parent {};
let _parent_from_child: child_proto::Parent = parent_proto::Parent::new();
}
#[test]
fn test_parent_serialization() {
assert_eq!(*parent_proto::Parent::new().serialize(), []);
}
#[test]
fn test_child_serialization() {
assert_eq!(*child_proto::Child::new().serialize(), []);
}

@ -0,0 +1,33 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Test input file.
int main() { return 0; }

@ -0,0 +1,33 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Test input file.
fn main() {}

@ -2,6 +2,7 @@
load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts")
load(":defs.bzl", "ActionsInfo", "attach_aspect")
load("//rust:defs.bzl", "RustProtoInfo")
def _find_action_with_mnemonic(actions, mnemonic):
action = [a for a in actions if a.mnemonic == mnemonic]
@ -21,56 +22,58 @@ def _find_rust_lib_input(inputs, target_name):
))
return input[0]
def _relevant_linker_inputs(ltl):
return ltl.objects + ltl.pic_objects + (
[ltl.static_library] if ltl.static_library else []
) + (
[ltl.pic_static_library] if ltl.pic_static_library else []
)
def _find_linker_input(rust_proto_info, basename_substring):
cc_info = rust_proto_info.dep_variant_info.cc_info
for linker_input in cc_info.linking_context.linker_inputs.to_list():
for ltl in linker_input.libraries:
for file in _relevant_linker_inputs(ltl):
if basename_substring in file.basename:
return file
fail("Couldn't find file with suffix {} in {}".format(
basename_substring,
[
f.basename
for input in cc_info.linking_context.linker_inputs.to_list()
for ltl in input.libraries
for f in _relevant_linker_inputs(ltl)
],
))
####################################################################################################
def _rust_compilation_action_has_runtime_as_input_test_impl(ctx):
def _rust_aspect_test_impl(ctx):
env = analysistest.begin(ctx)
target_under_test = analysistest.target_under_test(env)
actions = target_under_test[ActionsInfo].actions
rustc_action = _find_action_with_mnemonic(actions, "Rustc")
_find_rust_lib_input(rustc_action.inputs, "protobuf")
asserts.true(env, rustc_action.outputs.to_list()[0].path.endswith(".rlib"))
return analysistest.end(env)
rust_compilation_action_has_runtime_as_input_test = analysistest.make(
_rust_compilation_action_has_runtime_as_input_test_impl,
)
def _test_rust_compilation_action_has_runtime_as_input():
native.proto_library(name = "some_proto", srcs = ["some_proto.proto"])
attach_aspect(name = "some_proto_with_aspect", dep = ":some_proto")
rust_compilation_action_has_runtime_as_input_test(
name = "rust_compilation_action_has_runtime_as_input_test",
target_under_test = ":some_proto_with_aspect",
# TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain.
tags = ["not_build:arm"],
)
# The action needs to have the Rust runtime as an input
_find_rust_lib_input(rustc_action.inputs, "protobuf")
####################################################################################################
# The action needs to produce a .rlib artifact (sometimes .rmeta as well, not tested here).
asserts.true(env, rustc_action.outputs.to_list()[0].path.endswith(".rlib"))
def _rust_compilation_action_has_deps_as_inputs_test_impl(ctx):
env = analysistest.begin(ctx)
target_under_test = analysistest.target_under_test(env)
actions = target_under_test[ActionsInfo].actions
rustc_action = _find_action_with_mnemonic(actions, "Rustc")
_find_rust_lib_input(rustc_action.inputs, "parent")
# The aspect needs to provide CcInfo that passes UPB gencode to the eventual linking.
_find_linker_input(target_under_test[RustProtoInfo], "child.upb.thunks")
_find_linker_input(target_under_test[RustProtoInfo], "parent.upb.thunks")
return analysistest.end(env)
rust_compilation_action_has_deps_as_input_test = analysistest.make(
_rust_compilation_action_has_deps_as_inputs_test_impl,
)
def _test_rust_compilation_action_has_deps_as_input():
native.proto_library(name = "parent_proto", srcs = ["parent.proto"])
native.proto_library(name = "child_proto", srcs = ["child.proto"], deps = [":parent_proto"])
rust_aspect_test = analysistest.make(_rust_aspect_test_impl)
def _test_aspect():
attach_aspect(name = "child_proto_with_aspect", dep = ":child_proto")
rust_compilation_action_has_deps_as_input_test(
name = "rust_compilation_action_has_deps_as_input_test",
rust_aspect_test(
name = "rust_aspect_test",
target_under_test = ":child_proto_with_aspect",
# TODO(b/270274576): Enable testing on arm once we have a Rust Arm toolchain.
tags = ["not_build:arm"],
@ -83,13 +86,14 @@ def rust_proto_library_unit_test(name):
Args:
name: name of the test suite"""
_test_rust_compilation_action_has_runtime_as_input()
_test_rust_compilation_action_has_deps_as_input()
native.proto_library(name = "parent_proto", srcs = ["parent.proto"])
native.proto_library(name = "child_proto", srcs = ["child.proto"], deps = [":parent_proto"])
_test_aspect()
native.test_suite(
name = name,
tests = [
":rust_compilation_action_has_runtime_as_input_test",
":rust_compilation_action_has_deps_as_input_test",
":rust_aspect_test",
],
)

@ -28,8 +28,8 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
fn main() {
// This is currently just a smoke test checking that we can generate gencode, compile it, and
// link the test binary.
let _test_all_types: unittest_proto::TestAllTypes;
#[test]
fn test_serialization() {
let test_all_types: unittest_proto::TestAllTypes = unittest_proto::TestAllTypes::new();
assert_eq!(*test_all_types.serialize(), []);
}

@ -0,0 +1,27 @@
# This package contains Rust protobuf runtime implementation built on top of UPB.
load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
rust_library(
name = "upb",
srcs = ["upb.rs"],
visibility = ["//src/google/protobuf:__subpackages__"],
deps = [":upb_c_api"],
)
rust_test(
name = "upb_test",
crate = ":upb",
tags = [
"not_build:arm",
"notsan",
],
)
cc_library(
name = "upb_c_api",
srcs = ["upb_api.c"],
deps = [
"//third_party/upb",
],
)

@ -0,0 +1,53 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Rust bindings for UPB
#[repr(C)]
pub struct upb_Arena {
_data: [u8; 0],
_marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>,
}
extern "C" {
pub fn upb_Arena_New() -> *mut upb_Arena;
pub fn upb_Arena_Free(arena: *mut upb_Arena);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_arena_new_and_free() {
let arena = unsafe { upb_Arena_New() };
unsafe { upb_Arena_Free(arena) };
}
}

@ -0,0 +1,30 @@
/*
* Copyright (c) 2009-2021, Google LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Google LLC nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#define UPB_BUILD_API
#include "third_party/upb/upb/mem/arena.h" // IWYU pragma: keep

@ -89,15 +89,11 @@ bool RustGenerator::Generate(const FileDescriptor* file,
// TODO(b/270138878): Remove `do_nothing` import once we have real logic. This
// is there only to smoke test rustc actions in rust_proto_library.
p.Emit(R"rs(
#[allow(unused_imports)]
use protobuf::do_nothing;
extern crate protobuf as __pb;
extern crate std as __std;
)rs");
for (int i = 0; i < file->message_type_count(); ++i) {
// TODO(b/270138878): Implement real logic
p.Emit({{"Msg", file->message_type(i)->name()}}, R"rs(
pub struct $Msg$ {}
)rs");
}
// TODO(b/270124215): Delete the following "placeholder impl" of `import
// public`. Also make sure to figure out how to map FileDescriptor#name to
// Rust crate names (currently Bazel labels).
@ -114,6 +110,41 @@ bool RustGenerator::Generate(const FileDescriptor* file,
}
}
for (int i = 0; i < file->message_type_count(); ++i) {
// TODO(b/270138878): Implement real logic
std::string full_name = file->message_type(i)->full_name();
absl::StrReplaceAll({{".", "_"}}, &full_name);
p.Emit({{"Msg", file->message_type(i)->name()}, {"pkg_Msg", full_name}},
R"rs(
pub struct $Msg$ {
msg: ::__std::ptr::NonNull<u8>,
arena: *mut ::__pb::upb_Arena,
}
impl $Msg$ {
pub fn new() -> $Msg$ {
let arena = unsafe { ::__pb::upb_Arena_New() };
let msg = unsafe { $pkg_Msg$_new(arena) };
$Msg$ { msg, arena }
}
pub fn serialize(&self) -> ::__pb::SerializedData {
let arena = unsafe { ::__pb::upb_Arena_New() };
let mut len = 0;
let chars = unsafe { $pkg_Msg$_serialize(self.msg, arena, &mut len) };
unsafe {::__pb::SerializedData::from_raw_parts(arena, chars, len)}
}
}
extern "C" {
fn $pkg_Msg$_new(arena: *mut ::__pb::upb_Arena) -> ::__std::ptr::NonNull<u8>;
fn $pkg_Msg$_serialize(
msg: ::__std::ptr::NonNull<u8>,
arena: *mut ::__pb::upb_Arena,
len: &mut usize) -> ::__std::ptr::NonNull<u8>;
}
)rs");
}
return true;
}

Loading…
Cancel
Save