diff --git a/kokoro/common/bazel_flags.sh b/kokoro/common/bazel_flags.sh new file mode 100755 index 0000000000..6a2777fd25 --- /dev/null +++ b/kokoro/common/bazel_flags.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Helper for setting up common bazel flags in Kokoro. +# +# This script prints extra flags to a bazel invocation when it is run from +# Kokoro. When the special environment variables are not present (e.g., if you +# run Kokoro build scripts locally), this script only flips some debug settings. +# +# Example of running directly: +# bazel test $(path/to/bazel_flags.sh) //... + +function bazel_flags::gen_invocation_id() { + # Create a new invocation ID and store in the artifacts dir. + local _invocation_id=$(uuidgen | tr A-Z a-z) + + # Put the new invocation ID at the start of the output IDs file. Some + # Google-internal tools only look at the first entry, so this ensures the most + # recent entry is first. + local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids + local _temp_ids=$(mktemp) + echo ${_invocation_id} > ${_temp_ids} + [[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} + mv -f ${_temp_ids} ${_ids_file} + + echo -n ${_invocation_id} +} + +# Prints flags to use on Kokoro. +function bazel_flags::kokoro_flags() { + [[ -n ${KOKORO_JOB_NAME:-} ]] || return + + local -a _flags + _flags+=( + --invocation_id=$(bazel_flags::gen_invocation_id) + --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME} + ) + if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then + _flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) + else + _flags+=( --google_default_credentials=true ) + fi + + echo "${_flags[@]}" +} + +echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors" diff --git a/kokoro/common/bazel_wrapper.sh b/kokoro/common/bazel_wrapper.sh deleted file mode 100755 index bd35396077..0000000000 --- a/kokoro/common/bazel_wrapper.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Wrapper for invoking bazel on Kokoro. -# -# This script adds extra flags to a bazel invocation when it is run from Kokoro. -# When the special environment variables are not present (e.g., if you run -# Kokoro build scripts locally), this script is equivalent to the "bazel" -# command. -# -# Example of running directly: -# path/to/bazel_wrapper.sh build //... -# -# Example of `source`ing: -# source path/to/bazel_wrapper.sh -# bazel_wrapper build //... - -function bazel_wrapper::gen_invocation_id() { - # Create a new invocation ID and store in the artifacts dir. - local _invocation_id=$(uuidgen | tr A-Z a-z) - - # Put the new invocation ID at the start of the output IDs file. Some - # Google-internal tools only look at the first entry, so this ensures the most - # recent entry is first. - local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids - local _temp_ids=$(mktemp) - echo ${_invocation_id} > ${_temp_ids} - [[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} - mv -f ${_temp_ids} ${_ids_file} - - echo -n ${_invocation_id} -} - -# Prints flags to use on Kokoro. -function bazel_wrapper::kokoro_flags() { - [[ -n ${KOKORO_BES_PROJECT_ID:-} ]] || return - - local -a _flags - _flags+=( - --bes_backend=${KOKORO_BES_BACKEND_ADDRESS:-buildeventservice.googleapis.com} - --bes_results_url=https://source.cloud.google.com/results/invocations/ - --invocation_id=$(bazel_wrapper::gen_invocation_id) - --project_id=${KOKORO_BES_PROJECT_ID} # --bes_instance_name in Bazel 5+ - --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache - ) - if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then - _flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) - else - _flags+=( --google_default_credentials=true ) - fi - - echo "${_flags[@]}" -} - -# Runs bazel with Kokoro flags, if appropriate. -function bazel_wrapper() { - local -a _flags - - # We might need to add flags. They need to come after any startup flags and - # the command, but before any terminating "--", so copy them into the _flags - # variable. - until (( ${#@} == 0 )) || [[ $1 == "--" ]]; do - _flags+=( "${1}" ); shift - done - - # Set the `BAZEL` env variable to override the actual bazel binary to use: - ${BAZEL:=bazel} "${_flags[@]}" $(bazel_wrapper::kokoro_flags) "${@}" -} - -# If this script was called directly, run bazel. Otherwise (i.e., this script -# was `source`d), the sourcing script will call bazel_wrapper themselves. -(( ${#BASH_SOURCE[@]} == 1 )) && bazel_wrapper "${@}" diff --git a/kokoro/linux/bazel.sh b/kokoro/linux/bazel.sh index d97bb3b005..9f1656978e 100755 --- a/kokoro/linux/bazel.sh +++ b/kokoro/linux/bazel.sh @@ -39,8 +39,7 @@ function run { -v $GIT_REPO_ROOT:/workspace \ $CONTAINER_IMAGE \ test \ - --keep_going \ - --test_output=streamed \ + $(${GIT_REPO_ROOT}/kokoro/common/bazel_flags.sh) \ ${ENVS[@]} \ $PLATFORM_CONFIG \ $BAZEL_CONFIG \ diff --git a/kokoro/macos/cpp/build.sh b/kokoro/macos/cpp/build.sh index 7c21d4856d..9c3b8aa156 100755 --- a/kokoro/macos/cpp/build.sh +++ b/kokoro/macos/cpp/build.sh @@ -14,4 +14,4 @@ source kokoro/macos/prepare_build_macos_rc # # Run build # -bazel test //src/... -k --test_output=streamed +bazel test $(kokoro/common/bazel_flags.sh) //src/... diff --git a/kokoro/macos/cpp/common.cfg b/kokoro/macos/cpp/common.cfg index 4bea1cbbb1..26515517a9 100644 --- a/kokoro/macos/cpp/common.cfg +++ b/kokoro/macos/cpp/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/cpp/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/objectivec_ios_debug/common.cfg b/kokoro/macos/objectivec_ios_debug/common.cfg index 473d54554a..b4ccc45831 100644 --- a/kokoro/macos/objectivec_ios_debug/common.cfg +++ b/kokoro/macos/objectivec_ios_debug/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/objectivec_ios_debug/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/objectivec_ios_release/common.cfg b/kokoro/macos/objectivec_ios_release/common.cfg index 3cbfb685d8..3a6ac86716 100644 --- a/kokoro/macos/objectivec_ios_release/common.cfg +++ b/kokoro/macos/objectivec_ios_release/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/objectivec_ios_release/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/objectivec_osx/common.cfg b/kokoro/macos/objectivec_osx/common.cfg index 41bd46aa87..ee5c2db34f 100644 --- a/kokoro/macos/objectivec_osx/common.cfg +++ b/kokoro/macos/objectivec_osx/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/objectivec_osx/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/php74/common.cfg b/kokoro/macos/php74/common.cfg index cf7e80beb8..b9d5b822d6 100644 --- a/kokoro/macos/php74/common.cfg +++ b/kokoro/macos/php74/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/php74/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/php80/common.cfg b/kokoro/macos/php80/common.cfg index ded43e62c5..294fd52a00 100644 --- a/kokoro/macos/php80/common.cfg +++ b/kokoro/macos/php80/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/php80/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/python/build.sh b/kokoro/macos/python/build.sh index f1ea3aaf12..de3db88c67 100755 --- a/kokoro/macos/python/build.sh +++ b/kokoro/macos/python/build.sh @@ -9,4 +9,5 @@ cd $(dirname $0)/../../.. KOKORO_INSTALL_VENV=yes source kokoro/macos/prepare_build_macos_rc -bazel test //python/... @upb//python/... -k --macos_minimum_os=10.9 --test_output=streamed +bazel test //python/... @upb//python/... $(kokoro/common/bazel_flags.sh) \ + --macos_minimum_os=10.9 diff --git a/kokoro/macos/python/common.cfg b/kokoro/macos/python/common.cfg index ac3cbeebf5..dd9f8ac00a 100644 --- a/kokoro/macos/python/common.cfg +++ b/kokoro/macos/python/common.cfg @@ -9,3 +9,25 @@ action { regex: "**/*" } } + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/python_cpp/build.sh b/kokoro/macos/python_cpp/build.sh index ae480a5a06..8f94fcf9a8 100755 --- a/kokoro/macos/python_cpp/build.sh +++ b/kokoro/macos/python_cpp/build.sh @@ -9,4 +9,5 @@ cd $(dirname $0)/../../.. KOKORO_INSTALL_VENV=yes source kokoro/macos/prepare_build_macos_rc -bazel test //python/... -k --macos_minimum_os=10.9 --test_output=streamed --define=use_fast_cpp_protos=true +bazel test //python/... $(kokoro/common/bazel_flags.sh) \ + --macos_minimum_os=10.9 --define=use_fast_cpp_protos=true diff --git a/kokoro/macos/python_cpp/common.cfg b/kokoro/macos/python_cpp/common.cfg index 22f4a0e4c9..12270c531f 100644 --- a/kokoro/macos/python_cpp/common.cfg +++ b/kokoro/macos/python_cpp/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/python_cpp/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/ruby25/common.cfg b/kokoro/macos/ruby25/common.cfg index b3755e4104..ba17345259 100644 --- a/kokoro/macos/ruby25/common.cfg +++ b/kokoro/macos/ruby25/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/ruby25/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/ruby26/common.cfg b/kokoro/macos/ruby26/common.cfg index 688f63c2bc..deeaf51932 100644 --- a/kokoro/macos/ruby26/common.cfg +++ b/kokoro/macos/ruby26/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/ruby26/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/ruby27/common.cfg b/kokoro/macos/ruby27/common.cfg index b10b455da3..1451eb04ed 100644 --- a/kokoro/macos/ruby27/common.cfg +++ b/kokoro/macos/ruby27/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/ruby27/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/ruby30/common.cfg b/kokoro/macos/ruby30/common.cfg index d5051170be..79dc7a531a 100644 --- a/kokoro/macos/ruby30/common.cfg +++ b/kokoro/macos/ruby30/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/ruby30/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/ruby31/common.cfg b/kokoro/macos/ruby31/common.cfg index 19e16b3eaf..e2d1d1b83e 100644 --- a/kokoro/macos/ruby31/common.cfg +++ b/kokoro/macos/ruby31/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/macos/ruby31/build.sh" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/kokoro/macos/test_php.sh b/kokoro/macos/test_php.sh index 933b251101..74d88f6a36 100755 --- a/kokoro/macos/test_php.sh +++ b/kokoro/macos/test_php.sh @@ -11,4 +11,6 @@ composer test_c popd git clean -fXd -bazel test //php:conformance_test_c --action_env=PATH --test_env=PATH --test_output=streamed +bazel test $(kokoro/common/bazel_flags.sh) \ + --action_env=PATH --test_env=PATH \ + //php:conformance_test_c diff --git a/kokoro/windows/bazel/build.bat b/kokoro/windows/bazel/build.bat index 9b4e448f18..a783a075dd 100644 --- a/kokoro/windows/bazel/build.bat +++ b/kokoro/windows/bazel/build.bat @@ -13,9 +13,17 @@ bazel version choco install bazel -y -i --version 5.1.0 bazel version +@rem Set invocation ID so that bazel run is known to kokoro +uuidgen > %KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids +SET /p BAZEL_INTERNAL_INVOCATION_ID=<%KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids + @rem Make paths as short as possible to avoid long path issues. set BAZEL_STARTUP=--output_user_root=C:/tmp --windows_enable_symlinks -set BAZEL_FLAGS=--enable_runfiles --keep_going --test_output=streamed --verbose_failures +set BAZEL_FLAGS=--enable_runfiles --keep_going --test_output=errors ^ + --verbose_failures ^ + --invocation_id=%BAZEL_INTERNAL_INVOCATION_ID% ^ + --google_credentials=%KOKORO_BAZEL_AUTH_CREDENTIAL% ^ + --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/%KOKORO_JOB_NAME% @rem Build libraries first. bazel %BAZEL_STARTUP% build //:protoc //:protobuf //:protobuf_lite %BAZEL_FLAGS% || goto :error diff --git a/kokoro/windows/bazel/common.cfg b/kokoro/windows/bazel/common.cfg index 5978a7aa0d..52703cc16f 100644 --- a/kokoro/windows/bazel/common.cfg +++ b/kokoro/windows/bazel/common.cfg @@ -3,3 +3,25 @@ # Location of the build script in repository build_file: "protobuf/kokoro/windows/bazel/build.bat" timeout_mins: 1440 + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } + } +} +bazel_setting { + project_id: "protobuf-build" + bes_backend_address: "buildeventservice.googleapis.com" + foundry_backend_address: "remotebuildexecution.googleapis.com" + upsalite_frontend_address: "https://source.cloud.google.com" + local_execution: true + + # Need to be same as the fetch_keystore entry in the previous step. + auth_credential: { + keystore_config_id: 77103 + keyname: "kokoro_gcp_service" + } +} diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index 95e8e5b80f..afff93d3d8 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -8,7 +8,8 @@ set -eu # Some base locations. readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") readonly ProtoRootDir="${ScriptDir}/../.." -readonly BazelFlags="-k --announce_rc --test_output=errors --macos_minimum_os=10.9" +readonly BazelFlags="--announce_rc --macos_minimum_os=10.9 \ + $(${ScriptDir}/../../kokoro/common/bazel_flags.sh)" # Invoke with BAZEL=bazelisk to use that instead. readonly BazelBin="${BAZEL:=bazel}" diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index 6e9c612380..39f58a23d4 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -5,8 +5,8 @@ set -ex test_version() { version=$1 - bazel_args=" \ - -k --test_output=streamed \ + bazel_args="\ + $(../kokoro/common/bazel_flags.sh) \ --action_env=PATH \ --action_env=GEM_PATH \ --action_env=GEM_HOME \