Cross compiled protoc and protoc-gen-upb_minitable as part of the build process for creating the codegen crate.

Swap to tarballs instead of zips to preserve the +x flag on the executables

PiperOrigin-RevId: 681478567
pull/18573/head
Derek Benson 6 months ago committed by Copybara-Service
parent 1254c8daa0
commit 6bfff3909c
  1. 2
      .github/workflows/test_rust.yml
  2. 88
      rust/BUILD
  3. 24
      rust/cargo_test.sh
  4. 98
      rust/dist.bzl
  5. 9
      rust/protobuf_codegen/example/build.rs
  6. 66
      rust/protobuf_codegen/src/lib.rs

@ -40,4 +40,4 @@ jobs:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: rust_linux
bazel: >-
run //rust:cargo_test
run --crosstool_top=//toolchain:clang_suite --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //rust:cargo_test

@ -2,9 +2,10 @@
load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix")
load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain")
load("//rust:dist.bzl", "pkg_cross_compiled_binaries")
licenses(["notice"])
@ -284,8 +285,8 @@ pkg_filegroup(
prefix = "libupb",
)
pkg_zip(
name = "rust_crate",
pkg_tar(
name = "protobuf_crate_dist",
srcs = [
":crate_root_files",
":rust_protobuf_libupb_src",
@ -300,12 +301,25 @@ pkg_files(
strip_prefix = strip_prefix.from_root("rust/protobuf_codegen"),
)
pkg_zip(
name = "codegen_crate",
pkg_tar(
name = "codegen_crate_dist",
srcs = [
":protobuf_codegen_files",
":vendored_protocs_dist",
"//:LICENSE",
],
tags = ["manual"],
visibility = ["//rust:__pkg__"],
)
pkg_tar(
name = "codegen_crate_test",
srcs = [
":protobuf_codegen_files",
":vendored_protocs_test",
"//:LICENSE",
],
tags = ["manual"],
visibility = ["//rust:__pkg__"],
)
@ -315,8 +329,8 @@ pkg_files(
strip_prefix = strip_prefix.from_root("rust/protobuf_codegen/example"),
)
pkg_zip(
name = "codegen_example",
pkg_tar(
name = "codegen_example_test",
srcs = [
":codegen_example_files",
"//:LICENSE",
@ -324,15 +338,65 @@ pkg_zip(
visibility = ["//rust:__pkg__"],
)
# Bundle all protoc binaries for all platforms. Requires the toolchains to be installed.
pkg_cross_compiled_binaries(
name = "vendored_protocs_dist",
cpus = [
# TODO: Re-enable these platforms once the toolchains are available.
# "osx-x86_64",
# "osx-aarch_64",
"linux-aarch_64",
"linux-ppcle_64",
# "linux-s390_64",
"linux-x86_32",
"linux-x86_64",
"win32",
"win64",
],
prefix = "bin",
tags = ["manual"],
targets = [
"//upb_generator/minitable:protoc-gen-upb_minitable",
"//:protoc",
],
)
# Bundle only the linux-x86_64 protoc for testing.
pkg_cross_compiled_binaries(
name = "vendored_protocs_test",
cpus = [
"linux-x86_64",
],
prefix = "bin",
tags = ["manual"],
targets = [
"//upb_generator/minitable:protoc-gen-upb_minitable",
"//:protoc",
],
)
# Run the cargo test with only a bundled linux-x86_64 protoc.
sh_binary(
name = "cargo_test",
srcs = ["cargo_test.sh"],
data = [
":codegen_crate",
":codegen_example",
":rust_crate",
"//:protoc",
"//upb_generator/minitable:protoc-gen-upb_minitable",
":codegen_crate_test",
":codegen_example_test",
":protobuf_crate_dist",
],
tags = ["manual"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)
# Run the cargo test with all bundled protocs.
sh_binary(
name = "cargo_release_test",
srcs = ["cargo_test.sh"],
data = [
":codegen_crate_dist",
":codegen_example_test",
":protobuf_crate_dist",
],
tags = ["manual"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)

@ -35,23 +35,30 @@ mkdir $CARGO_HOME
CRATE_ROOT=$TMP_DIR/protobuf
mkdir $CRATE_ROOT
PROTOBUF_ZIP=$(rlocation com_google_protobuf/rust/rust_crate.zip)
PROTOBUF_TAR=$(rlocation com_google_protobuf/rust/protobuf_crate_dist.tar)
unzip -d $CRATE_ROOT $PROTOBUF_ZIP
echo "Expanding protobuf_crate tar"
tar -xvf $PROTOBUF_TAR -C $CRATE_ROOT
CODEGEN_ROOT=$TMP_DIR/protobuf_codegen
mkdir $CODEGEN_ROOT
CODEGEN_ZIP=$(rlocation com_google_protobuf/rust/codegen_crate.zip)
CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_dist.tar)
unzip -d $CODEGEN_ROOT $CODEGEN_ZIP
if [[ ! -f $CODEGEN_TAR ]]; then
CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_test.tar)
fi
echo "Expanding codegen_crate tar"
tar -xvf $CODEGEN_TAR -C $CODEGEN_ROOT
EXAMPLE_ROOT=$TMP_DIR/codegen_example
mkdir $EXAMPLE_ROOT
EXAMPLE_ZIP=$(rlocation com_google_protobuf/rust/codegen_example.zip)
EXAMPLE_TAR=$(rlocation com_google_protobuf/rust/codegen_example_test.tar)
unzip -d $EXAMPLE_ROOT $EXAMPLE_ZIP
echo "Expanding codegen_example tar"
tar -xvf $EXAMPLE_TAR -C $EXAMPLE_ROOT
cd $CRATE_ROOT
# Run all tests except doctests
@ -60,8 +67,5 @@ CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests
cd $CODEGEN_ROOT
CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests
PROTOC=$(rlocation com_google_protobuf/protoc)
PROTOC_GEN_UPB_MINITABLE=$(rlocation com_google_protobuf/upb_generator/minitable/protoc-gen-upb_minitable)
cd $EXAMPLE_ROOT
CARGO_HOME=$CARGO_HOME PROTOC=$PROTOC PROTOC_GEN_UPB_MINITABLE=$PROTOC_GEN_UPB_MINITABLE cargo test
CARGO_HOME=$CARGO_HOME cargo test

@ -0,0 +1,98 @@
"""
This module exports the pkg_cross_compiled_binaries rule. This rule is used to create a pkg_filegroup
that contains the cross compiled binaries for each cpu.
"""
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
def _cpu_transition_impl(settings, attr):
_ignore = (settings) # @unused
return [{"//command_line_option:cpu": attr.cpu}]
_cpu_transition = transition(
implementation = _cpu_transition_impl,
inputs = [],
outputs = [
"//command_line_option:cpu",
],
)
def _cross_compiled_binary_impl(ctx):
target = ctx.attr.target
default_info = target[0][DefaultInfo]
files = default_info.files
runfiles = default_info.default_runfiles
files = depset(transitive = [files])
return [
DefaultInfo(
files = files,
runfiles = runfiles,
),
]
_cross_compiled_binary = rule(
implementation = _cross_compiled_binary_impl,
attrs = {
"target": attr.label(
mandatory = True,
cfg = _cpu_transition,
),
"cpu": attr.string(),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
),
},
)
def pkg_cross_compiled_binaries(name, cpus, targets, prefix, tags):
"""Creates a pkg_filegroup that contains the cross compiled binaries for each cpu.
This rule is used to create a pkg_filegroup that contains the cross compiled binaries for each
cpu. The binaries are created by an aspect that changes the cpu configuration for each target.
The targets are placed into a directory that is named after the cpu.
Args:
name: The name of the pkg_filegroup.
cpus: The cpus to cross compile for.
targets: The targets to cross compile.
prefix: The prefix to add to the pkg_filegroup.
tags: The tags to add to the pkg_filegroup.
"""
filegroups = []
for cpu in cpus:
compiled_targets = []
for target in targets:
target_name = target.split(":")[1]
bin_target_name = name + "_" + cpu + "_" + target_name
_cross_compiled_binary(
name = bin_target_name,
cpu = cpu,
target = target,
)
compiled_targets.append(Label(":" + bin_target_name))
files_name = name + "_" + cpu + "_src"
filegroup_name = files_name + "_dir"
filegroups.append(Label(":" + filegroup_name))
pkg_files(
name = files_name,
srcs = compiled_targets,
attributes = pkg_attributes(
mode = "0755",
),
)
pkg_filegroup(
name = filegroup_name,
srcs = [Label(":" + files_name)],
prefix = cpu,
)
pkg_filegroup(
name = name,
srcs = filegroups,
prefix = prefix,
tags = tags,
)
return

@ -1,14 +1,7 @@
use protobuf_codegen::CodeGen;
use std::env;
fn main() {
let mut codegen = CodeGen::new();
codegen
.protoc_path(env::var("PROTOC").expect("PROTOC should be set to the path to protoc"))
.protoc_gen_upb_minitable_path(env::var("PROTOC_GEN_UPB_MINITABLE").expect(
"PROTOC_GEN_UPB_MINITABLE should be set to the path to protoc-gen-upb_minitable",
))
.inputs(["foo.proto", "bar/bar.proto"])
.include("proto");
codegen.inputs(["foo.proto", "bar/bar.proto"]).include("proto");
codegen.compile().unwrap();
}

@ -7,9 +7,8 @@ use walkdir::WalkDir;
pub struct CodeGen {
inputs: Vec<PathBuf>,
output_dir: PathBuf,
protoc_path: PathBuf,
protoc_gen_upb_path: PathBuf,
protoc_gen_upb_minitable_path: PathBuf,
protoc_path: Option<PathBuf>,
protoc_gen_upb_minitable_path: Option<PathBuf>,
includes: Vec<PathBuf>,
}
@ -20,9 +19,8 @@ impl CodeGen {
Self {
inputs: Vec::new(),
output_dir: std::env::current_dir().unwrap().join("src").join("protos"),
protoc_path: PathBuf::from("protoc"),
protoc_gen_upb_path: PathBuf::from("protoc-gen-upb"),
protoc_gen_upb_minitable_path: PathBuf::from("protoc-gen-upb_minitable"),
protoc_path: None,
protoc_gen_upb_minitable_path: None,
includes: Vec::new(),
}
}
@ -43,12 +41,7 @@ impl CodeGen {
}
pub fn protoc_path(&mut self, protoc_path: impl AsRef<Path>) -> &mut Self {
self.protoc_path = protoc_path.as_ref().to_owned();
self
}
pub fn protoc_gen_upb_path(&mut self, protoc_gen_upb_path: impl AsRef<Path>) -> &mut Self {
self.protoc_gen_upb_path = protoc_gen_upb_path.as_ref().to_owned();
self.protoc_path = Some(protoc_path.as_ref().to_owned());
self
}
@ -56,7 +49,8 @@ impl CodeGen {
&mut self,
protoc_gen_upb_minitable_path: impl AsRef<Path>,
) -> &mut Self {
self.protoc_gen_upb_minitable_path = protoc_gen_upb_minitable_path.as_ref().to_owned();
self.protoc_gen_upb_minitable_path =
Some(protoc_gen_upb_minitable_path.as_ref().to_owned());
self
}
@ -80,7 +74,12 @@ impl CodeGen {
);
}
let mut cmd = std::process::Command::new(&self.protoc_path);
let protoc_path = if let Some(path) = &self.protoc_path {
path.clone()
} else {
protoc_path().expect("To be a supported platform")
};
let mut cmd = std::process::Command::new(protoc_path);
for input in &self.inputs {
cmd.arg(input);
}
@ -88,12 +87,17 @@ impl CodeGen {
// Attempt to make the directory if it doesn't exist
let _ = std::fs::create_dir(&self.output_dir);
}
let protoc_gen_upb_minitable_path = if let Some(path) = &self.protoc_gen_upb_minitable_path
{
path.clone()
} else {
protoc_gen_upb_minitable_path().expect("To be a supported platform")
};
cmd.arg(format!("--rust_out={}", self.output_dir.display()))
.arg("--rust_opt=experimental-codegen=enabled,kernel=upb")
.arg(format!("--plugin=protoc-gen-upb={}", self.protoc_gen_upb_path.display()))
.arg(format!(
"--plugin=protoc-gen-upb_minitable={}",
self.protoc_gen_upb_minitable_path.display()
protoc_gen_upb_minitable_path.display()
))
.arg(format!("--upb_minitable_out={}", self.output_dir.display()));
for include in &self.includes {
@ -152,3 +156,33 @@ impl CodeGen {
file.write(contents.as_bytes()).unwrap();
}
}
fn get_path_for_arch() -> Option<PathBuf> {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.push("bin");
match (std::env::consts::OS, std::env::consts::ARCH) {
("macos", "x86_64") => path.push("osx-x86_64"),
("macos", "aarch64") => path.push("osx-aarch_64"),
("linux", "aarch64") => path.push("linux-aarch_64"),
("linux", "powerpc64") => path.push("linux-ppcle_64"),
("linux", "s390x") => path.push("linux-s390_64"),
("linux", "x86") => path.push("linux-x86_32"),
("linux", "x86_64") => path.push("linux-x86_64"),
("windows", "x86") => path.push("win32"),
("windows", "x86_64") => path.push("win64"),
_ => return None,
};
Some(path)
}
pub fn protoc_path() -> Option<PathBuf> {
let mut path = get_path_for_arch()?;
path.push("protoc");
Some(path)
}
pub fn protoc_gen_upb_minitable_path() -> Option<PathBuf> {
let mut path = get_path_for_arch()?;
path.push("protoc-gen-upb_minitable");
Some(path)
}

Loading…
Cancel
Save