Cross compiled protoc and protoc-gen-upb_minitable as part of the build process for creating the codegen crate.

Swap to tarballs instead of zips to preserve the +x flag on the executables

PiperOrigin-RevId: 681478567

.github/workflows/test_rust.yml

@@ -40,4 +40,4 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: rust_linux
           bazel: >-
-            run //rust:cargo_test
+            run --crosstool_top=//toolchain:clang_suite --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //rust:cargo_test

rust/BUILD

@@ -2,9 +2,10 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix")
-load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
+load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
 load("//bazel/toolchains:proto_lang_toolchain.bzl", "proto_lang_toolchain")
+load("//rust:dist.bzl", "pkg_cross_compiled_binaries")
 
 licenses(["notice"])
 
@@ -284,8 +285,8 @@ pkg_filegroup(
     prefix = "libupb",
 )
 
-pkg_zip(
+pkg_tar(
-    name = "rust_crate",
+    name = "protobuf_crate_dist",
     srcs = [
         ":crate_root_files",
         ":rust_protobuf_libupb_src",
@@ -300,12 +301,25 @@ pkg_files(
     strip_prefix = strip_prefix.from_root("rust/protobuf_codegen"),
 )
 
-pkg_zip(
+pkg_tar(
-    name = "codegen_crate",
+    name = "codegen_crate_dist",
     srcs = [
         ":protobuf_codegen_files",
+        ":vendored_protocs_dist",
         "//:LICENSE",
     ],
+    tags = ["manual"],
+    visibility = ["//rust:__pkg__"],
+)
+
+pkg_tar(
+    name = "codegen_crate_test",
+    srcs = [
+        ":protobuf_codegen_files",
+        ":vendored_protocs_test",
+        "//:LICENSE",
+    ],
+    tags = ["manual"],
     visibility = ["//rust:__pkg__"],
 )
 
@@ -315,8 +329,8 @@ pkg_files(
     strip_prefix = strip_prefix.from_root("rust/protobuf_codegen/example"),
 )
 
-pkg_zip(
+pkg_tar(
-    name = "codegen_example",
+    name = "codegen_example_test",
     srcs = [
         ":codegen_example_files",
         "//:LICENSE",
@@ -324,15 +338,65 @@ pkg_zip(
     visibility = ["//rust:__pkg__"],
 )
 
+# Bundle all protoc binaries for all platforms. Requires the toolchains to be installed.
+pkg_cross_compiled_binaries(
+    name = "vendored_protocs_dist",
+    cpus = [
+        # TODO: Re-enable these platforms once the toolchains are available.
+        # "osx-x86_64",
+        # "osx-aarch_64",
+        "linux-aarch_64",
+        "linux-ppcle_64",
+        # "linux-s390_64",
+        "linux-x86_32",
+        "linux-x86_64",
+        "win32",
+        "win64",
+    ],
+    prefix = "bin",
+    tags = ["manual"],
+    targets = [
+        "//upb_generator/minitable:protoc-gen-upb_minitable",
+        "//:protoc",
+    ],
+)
+
+# Bundle only the linux-x86_64 protoc for testing.
+pkg_cross_compiled_binaries(
+    name = "vendored_protocs_test",
+    cpus = [
+        "linux-x86_64",
+    ],
+    prefix = "bin",
+    tags = ["manual"],
+    targets = [
+        "//upb_generator/minitable:protoc-gen-upb_minitable",
+        "//:protoc",
+    ],
+)
+
+# Run the cargo test with only a bundled linux-x86_64 protoc.
 sh_binary(
     name = "cargo_test",
     srcs = ["cargo_test.sh"],
     data = [
-        ":codegen_crate",
+        ":codegen_crate_test",
-        ":codegen_example",
+        ":codegen_example_test",
-        ":rust_crate",
+        ":protobuf_crate_dist",
-        "//:protoc",
+    ],
-        "//upb_generator/minitable:protoc-gen-upb_minitable",
+    tags = ["manual"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
+# Run the cargo test with all bundled protocs.
+sh_binary(
+    name = "cargo_release_test",
+    srcs = ["cargo_test.sh"],
+    data = [
+        ":codegen_crate_dist",
+        ":codegen_example_test",
+        ":protobuf_crate_dist",
     ],
+    tags = ["manual"],
     deps = ["@bazel_tools//tools/bash/runfiles"],
 )

rust/cargo_test.sh

@@ -35,23 +35,30 @@ mkdir $CARGO_HOME
 CRATE_ROOT=$TMP_DIR/protobuf
 mkdir $CRATE_ROOT
 
-PROTOBUF_ZIP=$(rlocation com_google_protobuf/rust/rust_crate.zip)
+PROTOBUF_TAR=$(rlocation com_google_protobuf/rust/protobuf_crate_dist.tar)
 
-unzip -d $CRATE_ROOT $PROTOBUF_ZIP
+echo "Expanding protobuf_crate tar"
+tar -xvf $PROTOBUF_TAR -C $CRATE_ROOT 
 
 CODEGEN_ROOT=$TMP_DIR/protobuf_codegen
 mkdir $CODEGEN_ROOT
 
-CODEGEN_ZIP=$(rlocation com_google_protobuf/rust/codegen_crate.zip)
+CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_dist.tar)
 
-unzip -d $CODEGEN_ROOT $CODEGEN_ZIP
+if [[ ! -f $CODEGEN_TAR ]]; then
+    CODEGEN_TAR=$(rlocation com_google_protobuf/rust/codegen_crate_test.tar)
+fi
+
+echo "Expanding codegen_crate tar"
+tar -xvf $CODEGEN_TAR -C $CODEGEN_ROOT 
 
 EXAMPLE_ROOT=$TMP_DIR/codegen_example
 mkdir $EXAMPLE_ROOT
 
-EXAMPLE_ZIP=$(rlocation com_google_protobuf/rust/codegen_example.zip)
+EXAMPLE_TAR=$(rlocation com_google_protobuf/rust/codegen_example_test.tar)
 
-unzip -d $EXAMPLE_ROOT $EXAMPLE_ZIP
+echo "Expanding codegen_example tar"
+tar -xvf $EXAMPLE_TAR -C $EXAMPLE_ROOT 
 
 cd $CRATE_ROOT
 # Run all tests except doctests
@@ -60,8 +67,5 @@ CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests
 cd $CODEGEN_ROOT
 CARGO_HOME=$CARGO_HOME cargo test --lib --bins --tests
 
-PROTOC=$(rlocation com_google_protobuf/protoc)
-PROTOC_GEN_UPB_MINITABLE=$(rlocation com_google_protobuf/upb_generator/minitable/protoc-gen-upb_minitable)
-
 cd $EXAMPLE_ROOT
-CARGO_HOME=$CARGO_HOME PROTOC=$PROTOC PROTOC_GEN_UPB_MINITABLE=$PROTOC_GEN_UPB_MINITABLE cargo test
+CARGO_HOME=$CARGO_HOME cargo test

rust/dist.bzl

@@ -0,0 +1,98 @@
+"""
+This module exports the pkg_cross_compiled_binaries rule. This rule is used to create a pkg_filegroup
+that contains the cross compiled binaries for each cpu.
+"""
+
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+
+def _cpu_transition_impl(settings, attr):
+    _ignore = (settings)  # @unused
+    return [{"//command_line_option:cpu": attr.cpu}]
+
+_cpu_transition = transition(
+    implementation = _cpu_transition_impl,
+    inputs = [],
+    outputs = [
+        "//command_line_option:cpu",
+    ],
+)
+
+def _cross_compiled_binary_impl(ctx):
+    target = ctx.attr.target
+
+    default_info = target[0][DefaultInfo]
+    files = default_info.files
+    runfiles = default_info.default_runfiles
+
+    files = depset(transitive = [files])
+
+    return [
+        DefaultInfo(
+            files = files,
+            runfiles = runfiles,
+        ),
+    ]
+
+_cross_compiled_binary = rule(
+    implementation = _cross_compiled_binary_impl,
+    attrs = {
+        "target": attr.label(
+            mandatory = True,
+            cfg = _cpu_transition,
+        ),
+        "cpu": attr.string(),
+        "_allowlist_function_transition": attr.label(
+            default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+        ),
+    },
+)
+
+def pkg_cross_compiled_binaries(name, cpus, targets, prefix, tags):
+    """Creates a pkg_filegroup that contains the cross compiled binaries for each cpu.
+
+    This rule is used to create a pkg_filegroup that contains the cross compiled binaries for each
+    cpu. The binaries are created by an aspect that changes the cpu configuration for each target.
+    The targets are placed into a directory that is named after the cpu.
+
+    Args:
+      name: The name of the pkg_filegroup.
+      cpus: The cpus to cross compile for.
+      targets: The targets to cross compile.
+      prefix: The prefix to add to the pkg_filegroup.
+      tags: The tags to add to the pkg_filegroup.
+    """
+
+    filegroups = []
+    for cpu in cpus:
+        compiled_targets = []
+        for target in targets:
+            target_name = target.split(":")[1]
+            bin_target_name = name + "_" + cpu + "_" + target_name
+            _cross_compiled_binary(
+                name = bin_target_name,
+                cpu = cpu,
+                target = target,
+            )
+            compiled_targets.append(Label(":" + bin_target_name))
+        files_name = name + "_" + cpu + "_src"
+        filegroup_name = files_name + "_dir"
+        filegroups.append(Label(":" + filegroup_name))
+        pkg_files(
+            name = files_name,
+            srcs = compiled_targets,
+            attributes = pkg_attributes(
+                mode = "0755",
+            ),
+        )
+        pkg_filegroup(
+            name = filegroup_name,
+            srcs = [Label(":" + files_name)],
+            prefix = cpu,
+        )
+    pkg_filegroup(
+        name = name,
+        srcs = filegroups,
+        prefix = prefix,
+        tags = tags,
+    )
+    return

rust/protobuf_codegen/example/build.rs

@@ -1,14 +1,7 @@
 use protobuf_codegen::CodeGen;
-use std::env;
 
 fn main() {
     let mut codegen = CodeGen::new();
-    codegen
+    codegen.inputs(["foo.proto", "bar/bar.proto"]).include("proto");
-        .protoc_path(env::var("PROTOC").expect("PROTOC should be set to the path to protoc"))
-        .protoc_gen_upb_minitable_path(env::var("PROTOC_GEN_UPB_MINITABLE").expect(
-            "PROTOC_GEN_UPB_MINITABLE should be set to the path to protoc-gen-upb_minitable",
-        ))
-        .inputs(["foo.proto", "bar/bar.proto"])
-        .include("proto");
     codegen.compile().unwrap();
 }

rust/protobuf_codegen/src/lib.rs

@@ -7,9 +7,8 @@ use walkdir::WalkDir;
 pub struct CodeGen {
     inputs: Vec<PathBuf>,
     output_dir: PathBuf,
-    protoc_path: PathBuf,
+    protoc_path: Option<PathBuf>,
-    protoc_gen_upb_path: PathBuf,
+    protoc_gen_upb_minitable_path: Option<PathBuf>,
-    protoc_gen_upb_minitable_path: PathBuf,
     includes: Vec<PathBuf>,
 }
 
@@ -20,9 +19,8 @@ impl CodeGen {
         Self {
             inputs: Vec::new(),
             output_dir: std::env::current_dir().unwrap().join("src").join("protos"),
-            protoc_path: PathBuf::from("protoc"),
+            protoc_path: None,
-            protoc_gen_upb_path: PathBuf::from("protoc-gen-upb"),
+            protoc_gen_upb_minitable_path: None,
-            protoc_gen_upb_minitable_path: PathBuf::from("protoc-gen-upb_minitable"),
             includes: Vec::new(),
         }
     }
@@ -43,12 +41,7 @@ impl CodeGen {
     }
 
     pub fn protoc_path(&mut self, protoc_path: impl AsRef<Path>) -> &mut Self {
-        self.protoc_path = protoc_path.as_ref().to_owned();
+        self.protoc_path = Some(protoc_path.as_ref().to_owned());
-        self
-    }
-
-    pub fn protoc_gen_upb_path(&mut self, protoc_gen_upb_path: impl AsRef<Path>) -> &mut Self {
-        self.protoc_gen_upb_path = protoc_gen_upb_path.as_ref().to_owned();
         self
     }
 
@@ -56,7 +49,8 @@ impl CodeGen {
         &mut self,
         protoc_gen_upb_minitable_path: impl AsRef<Path>,
     ) -> &mut Self {
-        self.protoc_gen_upb_minitable_path = protoc_gen_upb_minitable_path.as_ref().to_owned();
+        self.protoc_gen_upb_minitable_path =
+            Some(protoc_gen_upb_minitable_path.as_ref().to_owned());
         self
     }
 
@@ -80,7 +74,12 @@ impl CodeGen {
             );
         }
 
-        let mut cmd = std::process::Command::new(&self.protoc_path);
+        let protoc_path = if let Some(path) = &self.protoc_path {
+            path.clone()
+        } else {
+            protoc_path().expect("To be a supported platform")
+        };
+        let mut cmd = std::process::Command::new(protoc_path);
         for input in &self.inputs {
             cmd.arg(input);
         }
@@ -88,12 +87,17 @@ impl CodeGen {
             // Attempt to make the directory if it doesn't exist
             let _ = std::fs::create_dir(&self.output_dir);
         }
+        let protoc_gen_upb_minitable_path = if let Some(path) = &self.protoc_gen_upb_minitable_path
+        {
+            path.clone()
+        } else {
+            protoc_gen_upb_minitable_path().expect("To be a supported platform")
+        };
         cmd.arg(format!("--rust_out={}", self.output_dir.display()))
             .arg("--rust_opt=experimental-codegen=enabled,kernel=upb")
-            .arg(format!("--plugin=protoc-gen-upb={}", self.protoc_gen_upb_path.display()))
             .arg(format!(
                 "--plugin=protoc-gen-upb_minitable={}",
-                self.protoc_gen_upb_minitable_path.display()
+                protoc_gen_upb_minitable_path.display()
             ))
             .arg(format!("--upb_minitable_out={}", self.output_dir.display()));
         for include in &self.includes {
@@ -152,3 +156,33 @@ impl CodeGen {
         file.write(contents.as_bytes()).unwrap();
     }
 }
+
+fn get_path_for_arch() -> Option<PathBuf> {
+    let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    path.push("bin");
+    match (std::env::consts::OS, std::env::consts::ARCH) {
+        ("macos", "x86_64") => path.push("osx-x86_64"),
+        ("macos", "aarch64") => path.push("osx-aarch_64"),
+        ("linux", "aarch64") => path.push("linux-aarch_64"),
+        ("linux", "powerpc64") => path.push("linux-ppcle_64"),
+        ("linux", "s390x") => path.push("linux-s390_64"),
+        ("linux", "x86") => path.push("linux-x86_32"),
+        ("linux", "x86_64") => path.push("linux-x86_64"),
+        ("windows", "x86") => path.push("win32"),
+        ("windows", "x86_64") => path.push("win64"),
+        _ => return None,
+    };
+    Some(path)
+}
+
+pub fn protoc_path() -> Option<PathBuf> {
+    let mut path = get_path_for_arch()?;
+    path.push("protoc");
+    Some(path)
+}
+
+pub fn protoc_gen_upb_minitable_path() -> Option<PathBuf> {
+    let mut path = get_path_for_arch()?;
+    path.push("protoc-gen-upb_minitable");
+    Some(path)
+}