diff --git a/.github/workflows/test_rust.yml b/.github/workflows/test_rust.yml index e081b82e17..5d81db5335 100644 --- a/.github/workflows/test_rust.yml +++ b/.github/workflows/test_rust.yml @@ -40,4 +40,4 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: rust_linux bazel: >- - run //rust:cargo_test + run --crosstool_top=//toolchain:clang_suite --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //rust:cargo_test diff --git a/rust/BUILD b/rust/BUILD index 8b6bb61b89..7e7be822bc 100644 --- a/rust/BUILD +++ b/rust/BUILD @@ -2,9 +2,10 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") -load("@rules_pkg//pkg:zip.bzl", "pkg_zip") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain") +load("//rust:dist.bzl", "pkg_cross_compiled_binaries") licenses(["notice"]) @@ -284,8 +285,8 @@ pkg_filegroup( prefix = "libupb", ) -pkg_zip( - name = "rust_crate", +pkg_tar( + name = "protobuf_crate_dist", srcs = [ ":crate_root_files", ":rust_protobuf_libupb_src", @@ -300,12 +301,25 @@ pkg_files( strip_prefix = strip_prefix.from_root("rust/protobuf_codegen"), ) -pkg_zip( - name = "codegen_crate", +pkg_tar( + name = "codegen_crate_dist", srcs = [ ":protobuf_codegen_files", + ":vendored_protocs_dist", "//:LICENSE", ], + tags = ["manual"], + visibility = ["//rust:__pkg__"], +) + +pkg_tar( + name = "codegen_crate_test", + srcs = [ + ":protobuf_codegen_files", + ":vendored_protocs_test", + "//:LICENSE", + ], + tags = ["manual"], visibility = ["//rust:__pkg__"], ) @@ -315,8 +329,8 @@ pkg_files( strip_prefix = strip_prefix.from_root("rust/protobuf_codegen/example"), ) -pkg_zip( - name = "codegen_example", +pkg_tar( + name = "codegen_example_test", srcs = [ ":codegen_example_files", "//:LICENSE", @@ -324,15 +338,65 @@ pkg_zip( visibility = ["//rust:__pkg__"], ) +# Bundle all protoc binaries for all platforms. Requires the toolchains to be installed. +pkg_cross_compiled_binaries( + name = "vendored_protocs_dist", + cpus = [ + # TODO: Re-enable these platforms once the toolchains are available. + # "osx-x86_64", + # "osx-aarch_64", + "linux-aarch_64", + "linux-ppcle_64", + # "linux-s390_64", + "linux-x86_32", + "linux-x86_64", + "win32", + "win64", + ], + prefix = "bin", + tags = ["manual"], + targets = [ + "//upb_generator/minitable:protoc-gen-upb_minitable", + "//:protoc", + ], +) + +# Bundle only the linux-x86_64 protoc for testing. +pkg_cross_compiled_binaries( + name = "vendored_protocs_test", + cpus = [ + "linux-x86_64", + ], + prefix = "bin", + tags = ["manual"], + targets = [ + "//upb_generator/minitable:protoc-gen-upb_minitable", + "//:protoc", + ], +) + +# Run the cargo test with only a bundled linux-x86_64 protoc. sh_binary( name = "cargo_test", srcs = ["cargo_test.sh"], data = [ - ":codegen_crate", - ":codegen_example", - ":rust_crate", - "//:protoc", - "//upb_generator/minitable:protoc-gen-upb_minitable", + ":codegen_crate_test", + ":codegen_example_test", + ":protobuf_crate_dist", + ], + tags = ["manual"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) + +# Run the cargo test with all bundled protocs. +sh_binary( + name = "cargo_release_test", + srcs = ["cargo_test.sh"], + data = [ + ":codegen_crate_dist", + ":codegen_example_test", + ":protobuf_crate_dist", ], + tags = ["manual"], deps = ["@bazel_tools//tools/bash/runfiles"], ) diff --git a/rust/cargo_test.sh b/rust/cargo_test.sh index 46460c9b3f..afc9adbe16 100755 --- a/rust/cargo_test.sh +++ b/rust/cargo_test.sh @@ -35,23 +35,30 @@ mkdir $CARGO_HOME CRATE_ROOT=$TMP_DIR/protobuf mkdir $CRATE_ROOT -PROTOBUF_ZIP=$(rlocation com_google_protobuf/rust/rust_crate.zip) +PROTOBUF_TAR=$(rlocation com_google_protobuf/rust/protobuf_crate_dist.tar) -unzip -d $CRATE_ROOT $PROTOBUF_ZIP +echo "Expanding protobuf_crate tar" +tar -xvf $PROTOBUF_TAR -C $CRATE_ROOT CODEGEN_ROOT=$TMP_DIR/protobuf_codegen mkdir $CODEGEN_ROOT -CODEGEN_ZIP=$(rlocation com_google_protobuf/rust/codegen_crate.zip) +CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_dist.tar) -unzip -d $CODEGEN_ROOT $CODEGEN_ZIP +if [[ ! -f $CODEGEN_TAR ]]; then + CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_test.tar) +fi + +echo "Expanding codegen_crate tar" +tar -xvf $CODEGEN_TAR -C $CODEGEN_ROOT EXAMPLE_ROOT=$TMP_DIR/codegen_example mkdir $EXAMPLE_ROOT -EXAMPLE_ZIP=$(rlocation com_google_protobuf/rust/codegen_example.zip) +EXAMPLE_TAR=$(rlocation com_google_protobuf/rust/codegen_example_test.tar) -unzip -d $EXAMPLE_ROOT $EXAMPLE_ZIP +echo "Expanding codegen_example tar" +tar -xvf $EXAMPLE_TAR -C $EXAMPLE_ROOT cd $CRATE_ROOT # Run all tests except doctests @@ -60,8 +67,5 @@ CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests cd $CODEGEN_ROOT CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests -PROTOC=$(rlocation com_google_protobuf/protoc) -PROTOC_GEN_UPB_MINITABLE=$(rlocation com_google_protobuf/upb_generator/minitable/protoc-gen-upb_minitable) - cd $EXAMPLE_ROOT -CARGO_HOME=$CARGO_HOME PROTOC=$PROTOC PROTOC_GEN_UPB_MINITABLE=$PROTOC_GEN_UPB_MINITABLE cargo test +CARGO_HOME=$CARGO_HOME cargo test diff --git a/rust/dist.bzl b/rust/dist.bzl new file mode 100644 index 0000000000..5ffb2898bc --- /dev/null +++ b/rust/dist.bzl @@ -0,0 +1,98 @@ +""" +This module exports the pkg_cross_compiled_binaries rule. This rule is used to create a pkg_filegroup +that contains the cross compiled binaries for each cpu. +""" + +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") + +def _cpu_transition_impl(settings, attr): + _ignore = (settings) # @unused + return [{"//command_line_option:cpu": attr.cpu}] + +_cpu_transition = transition( + implementation = _cpu_transition_impl, + inputs = [], + outputs = [ + "//command_line_option:cpu", + ], +) + +def _cross_compiled_binary_impl(ctx): + target = ctx.attr.target + + default_info = target[0][DefaultInfo] + files = default_info.files + runfiles = default_info.default_runfiles + + files = depset(transitive = [files]) + + return [ + DefaultInfo( + files = files, + runfiles = runfiles, + ), + ] + +_cross_compiled_binary = rule( + implementation = _cross_compiled_binary_impl, + attrs = { + "target": attr.label( + mandatory = True, + cfg = _cpu_transition, + ), + "cpu": attr.string(), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def pkg_cross_compiled_binaries(name, cpus, targets, prefix, tags): + """Creates a pkg_filegroup that contains the cross compiled binaries for each cpu. + + This rule is used to create a pkg_filegroup that contains the cross compiled binaries for each + cpu. The binaries are created by an aspect that changes the cpu configuration for each target. + The targets are placed into a directory that is named after the cpu. + + Args: + name: The name of the pkg_filegroup. + cpus: The cpus to cross compile for. + targets: The targets to cross compile. + prefix: The prefix to add to the pkg_filegroup. + tags: The tags to add to the pkg_filegroup. + """ + + filegroups = [] + for cpu in cpus: + compiled_targets = [] + for target in targets: + target_name = target.split(":")[1] + bin_target_name = name + "_" + cpu + "_" + target_name + _cross_compiled_binary( + name = bin_target_name, + cpu = cpu, + target = target, + ) + compiled_targets.append(Label(":" + bin_target_name)) + files_name = name + "_" + cpu + "_src" + filegroup_name = files_name + "_dir" + filegroups.append(Label(":" + filegroup_name)) + pkg_files( + name = files_name, + srcs = compiled_targets, + attributes = pkg_attributes( + mode = "0755", + ), + ) + pkg_filegroup( + name = filegroup_name, + srcs = [Label(":" + files_name)], + prefix = cpu, + ) + pkg_filegroup( + name = name, + srcs = filegroups, + prefix = prefix, + tags = tags, + ) + return diff --git a/rust/protobuf_codegen/example/build.rs b/rust/protobuf_codegen/example/build.rs index 4b80e1e8af..2b6a20204f 100644 --- a/rust/protobuf_codegen/example/build.rs +++ b/rust/protobuf_codegen/example/build.rs @@ -1,14 +1,7 @@ use protobuf_codegen::CodeGen; -use std::env; fn main() { let mut codegen = CodeGen::new(); - codegen - .protoc_path(env::var("PROTOC").expect("PROTOC should be set to the path to protoc")) - .protoc_gen_upb_minitable_path(env::var("PROTOC_GEN_UPB_MINITABLE").expect( - "PROTOC_GEN_UPB_MINITABLE should be set to the path to protoc-gen-upb_minitable", - )) - .inputs(["foo.proto", "bar/bar.proto"]) - .include("proto"); + codegen.inputs(["foo.proto", "bar/bar.proto"]).include("proto"); codegen.compile().unwrap(); } diff --git a/rust/protobuf_codegen/src/lib.rs b/rust/protobuf_codegen/src/lib.rs index db2998a0e3..2e3b8c57d4 100644 --- a/rust/protobuf_codegen/src/lib.rs +++ b/rust/protobuf_codegen/src/lib.rs @@ -7,9 +7,8 @@ use walkdir::WalkDir; pub struct CodeGen { inputs: Vec, output_dir: PathBuf, - protoc_path: PathBuf, - protoc_gen_upb_path: PathBuf, - protoc_gen_upb_minitable_path: PathBuf, + protoc_path: Option, + protoc_gen_upb_minitable_path: Option, includes: Vec, } @@ -20,9 +19,8 @@ impl CodeGen { Self { inputs: Vec::new(), output_dir: std::env::current_dir().unwrap().join("src").join("protos"), - protoc_path: PathBuf::from("protoc"), - protoc_gen_upb_path: PathBuf::from("protoc-gen-upb"), - protoc_gen_upb_minitable_path: PathBuf::from("protoc-gen-upb_minitable"), + protoc_path: None, + protoc_gen_upb_minitable_path: None, includes: Vec::new(), } } @@ -43,12 +41,7 @@ impl CodeGen { } pub fn protoc_path(&mut self, protoc_path: impl AsRef) -> &mut Self { - self.protoc_path = protoc_path.as_ref().to_owned(); - self - } - - pub fn protoc_gen_upb_path(&mut self, protoc_gen_upb_path: impl AsRef) -> &mut Self { - self.protoc_gen_upb_path = protoc_gen_upb_path.as_ref().to_owned(); + self.protoc_path = Some(protoc_path.as_ref().to_owned()); self } @@ -56,7 +49,8 @@ impl CodeGen { &mut self, protoc_gen_upb_minitable_path: impl AsRef, ) -> &mut Self { - self.protoc_gen_upb_minitable_path = protoc_gen_upb_minitable_path.as_ref().to_owned(); + self.protoc_gen_upb_minitable_path = + Some(protoc_gen_upb_minitable_path.as_ref().to_owned()); self } @@ -80,7 +74,12 @@ impl CodeGen { ); } - let mut cmd = std::process::Command::new(&self.protoc_path); + let protoc_path = if let Some(path) = &self.protoc_path { + path.clone() + } else { + protoc_path().expect("To be a supported platform") + }; + let mut cmd = std::process::Command::new(protoc_path); for input in &self.inputs { cmd.arg(input); } @@ -88,12 +87,17 @@ impl CodeGen { // Attempt to make the directory if it doesn't exist let _ = std::fs::create_dir(&self.output_dir); } + let protoc_gen_upb_minitable_path = if let Some(path) = &self.protoc_gen_upb_minitable_path + { + path.clone() + } else { + protoc_gen_upb_minitable_path().expect("To be a supported platform") + }; cmd.arg(format!("--rust_out={}", self.output_dir.display())) .arg("--rust_opt=experimental-codegen=enabled,kernel=upb") - .arg(format!("--plugin=protoc-gen-upb={}", self.protoc_gen_upb_path.display())) .arg(format!( "--plugin=protoc-gen-upb_minitable={}", - self.protoc_gen_upb_minitable_path.display() + protoc_gen_upb_minitable_path.display() )) .arg(format!("--upb_minitable_out={}", self.output_dir.display())); for include in &self.includes { @@ -152,3 +156,33 @@ impl CodeGen { file.write(contents.as_bytes()).unwrap(); } } + +fn get_path_for_arch() -> Option { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("bin"); + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "x86_64") => path.push("osx-x86_64"), + ("macos", "aarch64") => path.push("osx-aarch_64"), + ("linux", "aarch64") => path.push("linux-aarch_64"), + ("linux", "powerpc64") => path.push("linux-ppcle_64"), + ("linux", "s390x") => path.push("linux-s390_64"), + ("linux", "x86") => path.push("linux-x86_32"), + ("linux", "x86_64") => path.push("linux-x86_64"), + ("windows", "x86") => path.push("win32"), + ("windows", "x86_64") => path.push("win64"), + _ => return None, + }; + Some(path) +} + +pub fn protoc_path() -> Option { + let mut path = get_path_for_arch()?; + path.push("protoc"); + Some(path) +} + +pub fn protoc_gen_upb_minitable_path() -> Option { + let mut path = get_path_for_arch()?; + path.push("protoc-gen-upb_minitable"); + Some(path) +}