Mike Kruskal
2 years ago
committed by
Copybara-Service
parent
8dc14f688c
commit
eedef31738
8 changed files with 34 additions and 289 deletions
@ -1,46 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
# Helper for setting up common bazel flags in Kokoro. |
||||
# |
||||
# This script prints extra flags to a bazel invocation when it is run from |
||||
# Kokoro. When the special environment variables are not present (e.g., if you |
||||
# run Kokoro build scripts locally), this script only flips some debug settings. |
||||
# |
||||
# Example of running directly: |
||||
# bazel test $(path/to/bazel_flags.sh) //... |
||||
|
||||
function bazel_flags::gen_invocation_id() { |
||||
# Create a new invocation ID and store in the artifacts dir. |
||||
local _invocation_id=$(uuidgen | tr A-Z a-z) |
||||
|
||||
# Put the new invocation ID at the start of the output IDs file. Some |
||||
# Google-internal tools only look at the first entry, so this ensures the most |
||||
# recent entry is first. |
||||
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids |
||||
local _temp_ids=$(mktemp) |
||||
echo ${_invocation_id} > ${_temp_ids} |
||||
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} |
||||
mv -f ${_temp_ids} ${_ids_file} |
||||
|
||||
echo -n ${_invocation_id} |
||||
} |
||||
|
||||
# Prints flags to use on Kokoro. |
||||
function bazel_flags::kokoro_flags() { |
||||
[[ -n ${KOKORO_JOB_NAME:-} ]] || return |
||||
|
||||
local -a _flags |
||||
_flags+=( |
||||
--invocation_id=$(bazel_flags::gen_invocation_id) |
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME} |
||||
) |
||||
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then |
||||
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) |
||||
else |
||||
_flags+=( --google_default_credentials=true ) |
||||
fi |
||||
|
||||
echo "${_flags[@]}" |
||||
} |
||||
|
||||
echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors" |
||||
@ -1,92 +0,0 @@ |
||||
# Log capturing for the Kokoro runtime environment. |
||||
# |
||||
# This script should be `source`d from Kokoro build scripts to configure log |
||||
# capturing when running under Kokoro. |
||||
# |
||||
# When not running under Kokoro, no logs will be collected. If you want to run |
||||
# locally and collect logs anyway, set the KOKORO_ARTIFACTS_DIR environment |
||||
# variable to a directory where the logs should go. |
||||
# |
||||
# The job `.cfg` file needs the following stanzas to declare the captured logs |
||||
# as outputs (yes, these are globs, not regexes): |
||||
# |
||||
# action: { |
||||
# define_artifacts: { |
||||
# regex: "**/*sponge_log.log" |
||||
# regex: "**/*sponge_log.xml" |
||||
# } |
||||
# } |
||||
# |
||||
# Use the provided functions below as build/test fixtures, e.g.: |
||||
# |
||||
# source kokoro/common/capture_logs.sh |
||||
# caplog build/step1 <build command> |
||||
# caplog tests/step2 <test command> |
||||
# |
||||
# If log capturing is enabled, this script will set some variables that can be |
||||
# used if necessary: |
||||
# |
||||
# CAPLOG_DIR is used for logs |
||||
# CAPLOG_CMAKE_ARGS contains extra cmake args to enable test XML output |
||||
# CAPLOG_CTEST_ARGS contains extra ctest args to capture combined test logs |
||||
# |
||||
# For example: |
||||
# |
||||
# if [[ -v CAPLOG_DIR_BUILD ]]; then |
||||
# cp extra_diagnostics.log "${CAPLOG_DIR_BUILD}/diagnostics.log" |
||||
# fi |
||||
# |
||||
# # Use ${...:-} form under `set -u`: |
||||
# caplog build/01_configure cmake -G Ninja ${CAPLOG_CMAKE_ARGS:-} |
||||
# caplog build/02_build cmake --build |
||||
# caplog test/03_test ctest ${CAPLOG_CTEST_ARGS:-} |
||||
|
||||
if [[ -z ${KOKORO_ARTIFACTS_DIR:-} ]]; then |
||||
function caplog() { shift; "$@"; } |
||||
else |
||||
|
||||
CAPLOG_DIR="$(mktemp -d)" |
||||
CAPLOG_CMAKE_ARGS="-Dprotobuf_TEST_XML_OUTDIR=${CAPLOG_DIR}/tests/" |
||||
CAPLOG_CTEST_ARGS="--verbose" |
||||
|
||||
# Captures the stdout/stderr of a command to a named log file. |
||||
# Usage: caplog NAME COMMAND [ARGS...] |
||||
function caplog() { |
||||
_name="${CAPLOG_DIR}/${1%.log}.log"; shift |
||||
mkdir -p "${_name%/*}" |
||||
date |
||||
time ( "$@" 2>&1 | tee "${_name}" ) |
||||
if [[ $? != 0 ]] ; then |
||||
cat "${_name}" |
||||
return 1 |
||||
fi |
||||
} |
||||
|
||||
# Trap handler: renames logs on script exit so they will be found by Kokoro. |
||||
function _caplog_onexit() { |
||||
_rc=$? |
||||
set +x |
||||
echo "Collecting logs [${BASH_SOURCE}]" |
||||
|
||||
find "${CAPLOG_DIR}" -type f -name '*.log' \ |
||||
| while read _textlog; do |
||||
# Ensure an XML file exists for each .log file. |
||||
touch ${_textlog%.log}.xml |
||||
done |
||||
|
||||
find "${CAPLOG_DIR}" -type f \( -name '*.xml' -or -name '*.log' \) \ |
||||
| while read _src; do |
||||
# Move to artifacts dir, preserving the path relative to CAPLOG_DIR. |
||||
# The filename changes from foo/bar.log to foo/bar/sponge_log.log. |
||||
_logfile=${_src/${CAPLOG_DIR}\//} |
||||
_stem=${KOKORO_ARTIFACTS_DIR}/${_logfile%.*} |
||||
_ext=${_logfile##*.} |
||||
mkdir -p ${_stem} |
||||
mv -v "${_src}" "${_stem}/sponge_log.${_ext}" |
||||
done |
||||
rm -rv "${CAPLOG_DIR}" |
||||
exit ${_rc} |
||||
} |
||||
trap _caplog_onexit EXIT |
||||
|
||||
fi |
||||
@ -1,85 +0,0 @@ |
||||
#!/bin/bash |
||||
# |
||||
# Build tests under CMake. |
||||
# |
||||
# This script is used from macos and linux builds. It runs cmake and ctest in |
||||
# the current directory. Any additional setup should be done before running this |
||||
# script. |
||||
# |
||||
# This script uses `caplog` to save logfiles. See caplog.sh for details. |
||||
|
||||
set -eu -o pipefail |
||||
: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)} |
||||
|
||||
################################################################################ |
||||
# If you are using this script to run tests, you can set some environment |
||||
# variables to control behavior: |
||||
# |
||||
# By default, find the sources based on this script's path. |
||||
: ${SOURCE_DIR:=${SCRIPT_ROOT}} |
||||
# |
||||
# By default, put outputs under <git root>/cmake/build. |
||||
: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} |
||||
# |
||||
# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this |
||||
# variable explicitly for compatibility with older versions of cmake. See: |
||||
# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html |
||||
# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.) |
||||
if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then |
||||
CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" |
||||
else |
||||
CMAKE_BUILD_TYPE_FLAG= |
||||
fi |
||||
# |
||||
# For several other CMake options, see docs here: |
||||
# https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html |
||||
# |
||||
# Some variables you may want to override (see cmake docs for details): |
||||
# CMAKE_BUILD_PARALLEL_LEVEL |
||||
# CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE) |
||||
# CMAKE_GENERATOR |
||||
# CTEST_PARALLEL_LEVEL |
||||
################################################################################ |
||||
|
||||
echo "Building using..." |
||||
echo " Sources: ${SOURCE_DIR}" |
||||
echo " Build output: ${BUILD_DIR}" |
||||
if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then |
||||
echo " Build scripts: ${SCRIPT_ROOT}" |
||||
fi |
||||
set -x |
||||
source ${SCRIPT_ROOT}/kokoro/common/caplog.sh |
||||
|
||||
# |
||||
# Configure under $BUILD_DIR: |
||||
# |
||||
mkdir -p "${BUILD_DIR}" |
||||
|
||||
( |
||||
cd "${BUILD_DIR}" |
||||
caplog 01_configure \ |
||||
cmake -S "${SOURCE_DIR}" \ |
||||
${CMAKE_BUILD_TYPE_FLAG} \ |
||||
${CAPLOG_CMAKE_ARGS:-} \ |
||||
-DCMAKE_CXX_STANDARD=14 |
||||
) |
||||
if [[ -n ${CAPLOG_DIR:-} ]]; then |
||||
# Save configuration logs. |
||||
mkdir -p "${CAPLOG_DIR}/CMakeFiles" |
||||
cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" |
||||
fi |
||||
|
||||
# |
||||
# Build: |
||||
# |
||||
caplog 02_build \ |
||||
cmake --build "${BUILD_DIR}" |
||||
|
||||
# |
||||
# Run tests |
||||
# |
||||
( |
||||
cd "${BUILD_DIR}" |
||||
caplog 03_combined_testlog \ |
||||
ctest ${CAPLOG_CTEST_ARGS:-} |
||||
) |
||||
@ -1,11 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
set -eux |
||||
|
||||
# Upgrade to a supported gcc version |
||||
sudo add-apt-repository ppa:ubuntu-toolchain-r/test |
||||
sudo apt-get -y update && \ |
||||
sudo apt-get install --no-install-recommends -y --fix-missing --option Acquire::Retries=10 --option Acquire::http::Timeout="1800" \ |
||||
gcc-7 g++-7 |
||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 --slave /usr/bin/g++ g++ /usr/bin/g++-7 |
||||
sudo update-alternatives --set gcc /usr/bin/gcc-7 |
||||
@ -1,50 +0,0 @@ |
||||
@rem Update Chocolatey |
||||
choco upgrade -y --no-progress chocolatey |
||||
choco install -y python --version 3.10.0 |
||||
choco install -y --no-progress --pre cmake |
||||
|
||||
@rem Enable long paths. |
||||
Powershell.exe -Command "New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1 -PropertyType DWORD -Force" |
||||
|
||||
@rem Update git submodules. |
||||
git submodule update --init --recursive |
||||
|
||||
@rem Select Visual Studio 2017. |
||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64 |
||||
|
||||
@rem Convert Windows line breaks to Unix line breaks |
||||
@rem This allows text-matching tests to pass |
||||
@find . -type f -print0 | xargs -0 d2u |
||||
|
||||
@rem Use python3 |
||||
C:\python310\python.exe -m venv venv |
||||
call venv\Scripts\activate.bat |
||||
|
||||
@rem Setup Bazel |
||||
|
||||
@rem TODO(b/241475022) Use docker to guarantee better stability. |
||||
@rem Allow Bazel to create short paths. |
||||
fsutil 8dot3name set 0 |
||||
|
||||
@rem Reinstall Bazel due to corrupt installation in kokoro. |
||||
bazel version |
||||
choco install bazel -y -i --version 5.1.0 |
||||
bazel version |
||||
|
||||
@rem Set invocation ID so that bazel run is known to kokoro |
||||
uuidgen > %KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids |
||||
SET /p BAZEL_INTERNAL_INVOCATION_ID=<%KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids |
||||
|
||||
@rem Make paths as short as possible to avoid long path issues. |
||||
set BAZEL_STARTUP=--output_user_root=C:/tmp --windows_enable_symlinks |
||||
set BAZEL_FLAGS=--enable_runfiles --keep_going --test_output=errors ^ |
||||
--verbose_failures ^ |
||||
--invocation_id=%BAZEL_INTERNAL_INVOCATION_ID% ^ |
||||
--google_credentials=%KOKORO_BAZEL_AUTH_CREDENTIAL% ^ |
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/%KOKORO_JOB_NAME% |
||||
|
||||
@rem Regenerate stale CMake configs. |
||||
@rem LINT.IfChange(staleness_tests) |
||||
bazel test //src:cmake_lists_staleness_test || call bazel-bin\src\cmake_lists_staleness_test.exe --fix |
||||
bazel test //src/google/protobuf:well_known_types_staleness_test || call bazel-bin\src\google\protobuf\well_known_types_staleness_test.exe --fix |
||||
@rem LINT.ThenChange(//depot/google3/third_party/protobuf/github/regenerate_stale_files.sh) |
||||
Loading…
Reference in new issue