Add remote caching to bazel builds (#10767)
* Add remote cache to linux builds * Remove BES flags * Remove BES flags from the right file * Migrate all Bazel kokoro builds to use remote caching * Remove BES logic * Fix mac ruby tests * Give mac/windows builds GCP access * Adding quotes to prevent issues with common flags * Adding command echoing in windows builds * Try enabling command echoing again * Adding invocation id for windows bazel build * Third try * Adding credentials to windows buildpull/10786/head
Mike Kruskal
2 years ago
committed by
GitHub
parent
7d1362c7ba
commit
79564c53af
24 changed files with 376 additions and 81 deletions
@ -0,0 +1,46 @@ |
|||||||
|
#!/bin/bash |
||||||
|
|
||||||
|
# Helper for setting up common bazel flags in Kokoro. |
||||||
|
# |
||||||
|
# This script prints extra flags to a bazel invocation when it is run from |
||||||
|
# Kokoro. When the special environment variables are not present (e.g., if you |
||||||
|
# run Kokoro build scripts locally), this script only flips some debug settings. |
||||||
|
# |
||||||
|
# Example of running directly: |
||||||
|
# bazel test $(path/to/bazel_flags.sh) //... |
||||||
|
|
||||||
|
function bazel_flags::gen_invocation_id() { |
||||||
|
# Create a new invocation ID and store in the artifacts dir. |
||||||
|
local _invocation_id=$(uuidgen | tr A-Z a-z) |
||||||
|
|
||||||
|
# Put the new invocation ID at the start of the output IDs file. Some |
||||||
|
# Google-internal tools only look at the first entry, so this ensures the most |
||||||
|
# recent entry is first. |
||||||
|
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids |
||||||
|
local _temp_ids=$(mktemp) |
||||||
|
echo ${_invocation_id} > ${_temp_ids} |
||||||
|
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} |
||||||
|
mv -f ${_temp_ids} ${_ids_file} |
||||||
|
|
||||||
|
echo -n ${_invocation_id} |
||||||
|
} |
||||||
|
|
||||||
|
# Prints flags to use on Kokoro. |
||||||
|
function bazel_flags::kokoro_flags() { |
||||||
|
[[ -n ${KOKORO_JOB_NAME:-} ]] || return |
||||||
|
|
||||||
|
local -a _flags |
||||||
|
_flags+=( |
||||||
|
--invocation_id=$(bazel_flags::gen_invocation_id) |
||||||
|
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME} |
||||||
|
) |
||||||
|
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then |
||||||
|
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) |
||||||
|
else |
||||||
|
_flags+=( --google_default_credentials=true ) |
||||||
|
fi |
||||||
|
|
||||||
|
echo "${_flags[@]}" |
||||||
|
} |
||||||
|
|
||||||
|
echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors" |
||||||
@ -1,71 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
|
|
||||||
# Wrapper for invoking bazel on Kokoro. |
|
||||||
# |
|
||||||
# This script adds extra flags to a bazel invocation when it is run from Kokoro. |
|
||||||
# When the special environment variables are not present (e.g., if you run |
|
||||||
# Kokoro build scripts locally), this script is equivalent to the "bazel" |
|
||||||
# command. |
|
||||||
# |
|
||||||
# Example of running directly: |
|
||||||
# path/to/bazel_wrapper.sh build //... |
|
||||||
# |
|
||||||
# Example of `source`ing: |
|
||||||
# source path/to/bazel_wrapper.sh |
|
||||||
# bazel_wrapper build //... |
|
||||||
|
|
||||||
function bazel_wrapper::gen_invocation_id() { |
|
||||||
# Create a new invocation ID and store in the artifacts dir. |
|
||||||
local _invocation_id=$(uuidgen | tr A-Z a-z) |
|
||||||
|
|
||||||
# Put the new invocation ID at the start of the output IDs file. Some |
|
||||||
# Google-internal tools only look at the first entry, so this ensures the most |
|
||||||
# recent entry is first. |
|
||||||
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids |
|
||||||
local _temp_ids=$(mktemp) |
|
||||||
echo ${_invocation_id} > ${_temp_ids} |
|
||||||
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} |
|
||||||
mv -f ${_temp_ids} ${_ids_file} |
|
||||||
|
|
||||||
echo -n ${_invocation_id} |
|
||||||
} |
|
||||||
|
|
||||||
# Prints flags to use on Kokoro. |
|
||||||
function bazel_wrapper::kokoro_flags() { |
|
||||||
[[ -n ${KOKORO_BES_PROJECT_ID:-} ]] || return |
|
||||||
|
|
||||||
local -a _flags |
|
||||||
_flags+=( |
|
||||||
--bes_backend=${KOKORO_BES_BACKEND_ADDRESS:-buildeventservice.googleapis.com} |
|
||||||
--bes_results_url=https://source.cloud.google.com/results/invocations/ |
|
||||||
--invocation_id=$(bazel_wrapper::gen_invocation_id) |
|
||||||
--project_id=${KOKORO_BES_PROJECT_ID} # --bes_instance_name in Bazel 5+ |
|
||||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache |
|
||||||
) |
|
||||||
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then |
|
||||||
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) |
|
||||||
else |
|
||||||
_flags+=( --google_default_credentials=true ) |
|
||||||
fi |
|
||||||
|
|
||||||
echo "${_flags[@]}" |
|
||||||
} |
|
||||||
|
|
||||||
# Runs bazel with Kokoro flags, if appropriate. |
|
||||||
function bazel_wrapper() { |
|
||||||
local -a _flags |
|
||||||
|
|
||||||
# We might need to add flags. They need to come after any startup flags and |
|
||||||
# the command, but before any terminating "--", so copy them into the _flags |
|
||||||
# variable. |
|
||||||
until (( ${#@} == 0 )) || [[ $1 == "--" ]]; do |
|
||||||
_flags+=( "${1}" ); shift |
|
||||||
done |
|
||||||
|
|
||||||
# Set the `BAZEL` env variable to override the actual bazel binary to use: |
|
||||||
${BAZEL:=bazel} "${_flags[@]}" $(bazel_wrapper::kokoro_flags) "${@}" |
|
||||||
} |
|
||||||
|
|
||||||
# If this script was called directly, run bazel. Otherwise (i.e., this script |
|
||||||
# was `source`d), the sourcing script will call bazel_wrapper themselves. |
|
||||||
(( ${#BASH_SOURCE[@]} == 1 )) && bazel_wrapper "${@}" |
|
||||||
Loading…
Reference in new issue