Mike Kruskal
2 years ago
committed by
Copybara-Service
parent
8dc14f688c
commit
eedef31738
8 changed files with 34 additions and 289 deletions
@ -1,46 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
|
|
||||||
# Helper for setting up common bazel flags in Kokoro. |
|
||||||
# |
|
||||||
# This script prints extra flags to a bazel invocation when it is run from |
|
||||||
# Kokoro. When the special environment variables are not present (e.g., if you |
|
||||||
# run Kokoro build scripts locally), this script only flips some debug settings. |
|
||||||
# |
|
||||||
# Example of running directly: |
|
||||||
# bazel test $(path/to/bazel_flags.sh) //... |
|
||||||
|
|
||||||
function bazel_flags::gen_invocation_id() { |
|
||||||
# Create a new invocation ID and store in the artifacts dir. |
|
||||||
local _invocation_id=$(uuidgen | tr A-Z a-z) |
|
||||||
|
|
||||||
# Put the new invocation ID at the start of the output IDs file. Some |
|
||||||
# Google-internal tools only look at the first entry, so this ensures the most |
|
||||||
# recent entry is first. |
|
||||||
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids |
|
||||||
local _temp_ids=$(mktemp) |
|
||||||
echo ${_invocation_id} > ${_temp_ids} |
|
||||||
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} |
|
||||||
mv -f ${_temp_ids} ${_ids_file} |
|
||||||
|
|
||||||
echo -n ${_invocation_id} |
|
||||||
} |
|
||||||
|
|
||||||
# Prints flags to use on Kokoro. |
|
||||||
function bazel_flags::kokoro_flags() { |
|
||||||
[[ -n ${KOKORO_JOB_NAME:-} ]] || return |
|
||||||
|
|
||||||
local -a _flags |
|
||||||
_flags+=( |
|
||||||
--invocation_id=$(bazel_flags::gen_invocation_id) |
|
||||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME} |
|
||||||
) |
|
||||||
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then |
|
||||||
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) |
|
||||||
else |
|
||||||
_flags+=( --google_default_credentials=true ) |
|
||||||
fi |
|
||||||
|
|
||||||
echo "${_flags[@]}" |
|
||||||
} |
|
||||||
|
|
||||||
echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors" |
|
||||||
@ -1,92 +0,0 @@ |
|||||||
# Log capturing for the Kokoro runtime environment. |
|
||||||
# |
|
||||||
# This script should be `source`d from Kokoro build scripts to configure log |
|
||||||
# capturing when running under Kokoro. |
|
||||||
# |
|
||||||
# When not running under Kokoro, no logs will be collected. If you want to run |
|
||||||
# locally and collect logs anyway, set the KOKORO_ARTIFACTS_DIR environment |
|
||||||
# variable to a directory where the logs should go. |
|
||||||
# |
|
||||||
# The job `.cfg` file needs the following stanzas to declare the captured logs |
|
||||||
# as outputs (yes, these are globs, not regexes): |
|
||||||
# |
|
||||||
# action: { |
|
||||||
# define_artifacts: { |
|
||||||
# regex: "**/*sponge_log.log" |
|
||||||
# regex: "**/*sponge_log.xml" |
|
||||||
# } |
|
||||||
# } |
|
||||||
# |
|
||||||
# Use the provided functions below as build/test fixtures, e.g.: |
|
||||||
# |
|
||||||
# source kokoro/common/capture_logs.sh |
|
||||||
# caplog build/step1 <build command> |
|
||||||
# caplog tests/step2 <test command> |
|
||||||
# |
|
||||||
# If log capturing is enabled, this script will set some variables that can be |
|
||||||
# used if necessary: |
|
||||||
# |
|
||||||
# CAPLOG_DIR is used for logs |
|
||||||
# CAPLOG_CMAKE_ARGS contains extra cmake args to enable test XML output |
|
||||||
# CAPLOG_CTEST_ARGS contains extra ctest args to capture combined test logs |
|
||||||
# |
|
||||||
# For example: |
|
||||||
# |
|
||||||
# if [[ -v CAPLOG_DIR_BUILD ]]; then |
|
||||||
# cp extra_diagnostics.log "${CAPLOG_DIR_BUILD}/diagnostics.log" |
|
||||||
# fi |
|
||||||
# |
|
||||||
# # Use ${...:-} form under `set -u`: |
|
||||||
# caplog build/01_configure cmake -G Ninja ${CAPLOG_CMAKE_ARGS:-} |
|
||||||
# caplog build/02_build cmake --build |
|
||||||
# caplog test/03_test ctest ${CAPLOG_CTEST_ARGS:-} |
|
||||||
|
|
||||||
if [[ -z ${KOKORO_ARTIFACTS_DIR:-} ]]; then |
|
||||||
function caplog() { shift; "$@"; } |
|
||||||
else |
|
||||||
|
|
||||||
CAPLOG_DIR="$(mktemp -d)" |
|
||||||
CAPLOG_CMAKE_ARGS="-Dprotobuf_TEST_XML_OUTDIR=${CAPLOG_DIR}/tests/" |
|
||||||
CAPLOG_CTEST_ARGS="--verbose" |
|
||||||
|
|
||||||
# Captures the stdout/stderr of a command to a named log file. |
|
||||||
# Usage: caplog NAME COMMAND [ARGS...] |
|
||||||
function caplog() { |
|
||||||
_name="${CAPLOG_DIR}/${1%.log}.log"; shift |
|
||||||
mkdir -p "${_name%/*}" |
|
||||||
date |
|
||||||
time ( "$@" 2>&1 | tee "${_name}" ) |
|
||||||
if [[ $? != 0 ]] ; then |
|
||||||
cat "${_name}" |
|
||||||
return 1 |
|
||||||
fi |
|
||||||
} |
|
||||||
|
|
||||||
# Trap handler: renames logs on script exit so they will be found by Kokoro. |
|
||||||
function _caplog_onexit() { |
|
||||||
_rc=$? |
|
||||||
set +x |
|
||||||
echo "Collecting logs [${BASH_SOURCE}]" |
|
||||||
|
|
||||||
find "${CAPLOG_DIR}" -type f -name '*.log' \ |
|
||||||
| while read _textlog; do |
|
||||||
# Ensure an XML file exists for each .log file. |
|
||||||
touch ${_textlog%.log}.xml |
|
||||||
done |
|
||||||
|
|
||||||
find "${CAPLOG_DIR}" -type f \( -name '*.xml' -or -name '*.log' \) \ |
|
||||||
| while read _src; do |
|
||||||
# Move to artifacts dir, preserving the path relative to CAPLOG_DIR. |
|
||||||
# The filename changes from foo/bar.log to foo/bar/sponge_log.log. |
|
||||||
_logfile=${_src/${CAPLOG_DIR}\//} |
|
||||||
_stem=${KOKORO_ARTIFACTS_DIR}/${_logfile%.*} |
|
||||||
_ext=${_logfile##*.} |
|
||||||
mkdir -p ${_stem} |
|
||||||
mv -v "${_src}" "${_stem}/sponge_log.${_ext}" |
|
||||||
done |
|
||||||
rm -rv "${CAPLOG_DIR}" |
|
||||||
exit ${_rc} |
|
||||||
} |
|
||||||
trap _caplog_onexit EXIT |
|
||||||
|
|
||||||
fi |
|
||||||
@ -1,85 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# |
|
||||||
# Build tests under CMake. |
|
||||||
# |
|
||||||
# This script is used from macos and linux builds. It runs cmake and ctest in |
|
||||||
# the current directory. Any additional setup should be done before running this |
|
||||||
# script. |
|
||||||
# |
|
||||||
# This script uses `caplog` to save logfiles. See caplog.sh for details. |
|
||||||
|
|
||||||
set -eu -o pipefail |
|
||||||
: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)} |
|
||||||
|
|
||||||
################################################################################ |
|
||||||
# If you are using this script to run tests, you can set some environment |
|
||||||
# variables to control behavior: |
|
||||||
# |
|
||||||
# By default, find the sources based on this script's path. |
|
||||||
: ${SOURCE_DIR:=${SCRIPT_ROOT}} |
|
||||||
# |
|
||||||
# By default, put outputs under <git root>/cmake/build. |
|
||||||
: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} |
|
||||||
# |
|
||||||
# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this |
|
||||||
# variable explicitly for compatibility with older versions of cmake. See: |
|
||||||
# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html |
|
||||||
# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.) |
|
||||||
if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then |
|
||||||
CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" |
|
||||||
else |
|
||||||
CMAKE_BUILD_TYPE_FLAG= |
|
||||||
fi |
|
||||||
# |
|
||||||
# For several other CMake options, see docs here: |
|
||||||
# https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html |
|
||||||
# |
|
||||||
# Some variables you may want to override (see cmake docs for details): |
|
||||||
# CMAKE_BUILD_PARALLEL_LEVEL |
|
||||||
# CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE) |
|
||||||
# CMAKE_GENERATOR |
|
||||||
# CTEST_PARALLEL_LEVEL |
|
||||||
################################################################################ |
|
||||||
|
|
||||||
echo "Building using..." |
|
||||||
echo " Sources: ${SOURCE_DIR}" |
|
||||||
echo " Build output: ${BUILD_DIR}" |
|
||||||
if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then |
|
||||||
echo " Build scripts: ${SCRIPT_ROOT}" |
|
||||||
fi |
|
||||||
set -x |
|
||||||
source ${SCRIPT_ROOT}/kokoro/common/caplog.sh |
|
||||||
|
|
||||||
# |
|
||||||
# Configure under $BUILD_DIR: |
|
||||||
# |
|
||||||
mkdir -p "${BUILD_DIR}" |
|
||||||
|
|
||||||
( |
|
||||||
cd "${BUILD_DIR}" |
|
||||||
caplog 01_configure \ |
|
||||||
cmake -S "${SOURCE_DIR}" \ |
|
||||||
${CMAKE_BUILD_TYPE_FLAG} \ |
|
||||||
${CAPLOG_CMAKE_ARGS:-} \ |
|
||||||
-DCMAKE_CXX_STANDARD=14 |
|
||||||
) |
|
||||||
if [[ -n ${CAPLOG_DIR:-} ]]; then |
|
||||||
# Save configuration logs. |
|
||||||
mkdir -p "${CAPLOG_DIR}/CMakeFiles" |
|
||||||
cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" |
|
||||||
fi |
|
||||||
|
|
||||||
# |
|
||||||
# Build: |
|
||||||
# |
|
||||||
caplog 02_build \ |
|
||||||
cmake --build "${BUILD_DIR}" |
|
||||||
|
|
||||||
# |
|
||||||
# Run tests |
|
||||||
# |
|
||||||
( |
|
||||||
cd "${BUILD_DIR}" |
|
||||||
caplog 03_combined_testlog \ |
|
||||||
ctest ${CAPLOG_CTEST_ARGS:-} |
|
||||||
) |
|
||||||
@ -1,11 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
|
|
||||||
set -eux |
|
||||||
|
|
||||||
# Upgrade to a supported gcc version |
|
||||||
sudo add-apt-repository ppa:ubuntu-toolchain-r/test |
|
||||||
sudo apt-get -y update && \ |
|
||||||
sudo apt-get install --no-install-recommends -y --fix-missing --option Acquire::Retries=10 --option Acquire::http::Timeout="1800" \ |
|
||||||
gcc-7 g++-7 |
|
||||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 --slave /usr/bin/g++ g++ /usr/bin/g++-7 |
|
||||||
sudo update-alternatives --set gcc /usr/bin/gcc-7 |
|
||||||
@ -1,50 +0,0 @@ |
|||||||
@rem Update Chocolatey |
|
||||||
choco upgrade -y --no-progress chocolatey |
|
||||||
choco install -y python --version 3.10.0 |
|
||||||
choco install -y --no-progress --pre cmake |
|
||||||
|
|
||||||
@rem Enable long paths. |
|
||||||
Powershell.exe -Command "New-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name LongPathsEnabled -Value 1 -PropertyType DWORD -Force" |
|
||||||
|
|
||||||
@rem Update git submodules. |
|
||||||
git submodule update --init --recursive |
|
||||||
|
|
||||||
@rem Select Visual Studio 2017. |
|
||||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" x64 |
|
||||||
|
|
||||||
@rem Convert Windows line breaks to Unix line breaks |
|
||||||
@rem This allows text-matching tests to pass |
|
||||||
@find . -type f -print0 | xargs -0 d2u |
|
||||||
|
|
||||||
@rem Use python3 |
|
||||||
C:\python310\python.exe -m venv venv |
|
||||||
call venv\Scripts\activate.bat |
|
||||||
|
|
||||||
@rem Setup Bazel |
|
||||||
|
|
||||||
@rem TODO(b/241475022) Use docker to guarantee better stability. |
|
||||||
@rem Allow Bazel to create short paths. |
|
||||||
fsutil 8dot3name set 0 |
|
||||||
|
|
||||||
@rem Reinstall Bazel due to corrupt installation in kokoro. |
|
||||||
bazel version |
|
||||||
choco install bazel -y -i --version 5.1.0 |
|
||||||
bazel version |
|
||||||
|
|
||||||
@rem Set invocation ID so that bazel run is known to kokoro |
|
||||||
uuidgen > %KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids |
|
||||||
SET /p BAZEL_INTERNAL_INVOCATION_ID=<%KOKORO_ARTIFACTS_DIR%\bazel_invocation_ids |
|
||||||
|
|
||||||
@rem Make paths as short as possible to avoid long path issues. |
|
||||||
set BAZEL_STARTUP=--output_user_root=C:/tmp --windows_enable_symlinks |
|
||||||
set BAZEL_FLAGS=--enable_runfiles --keep_going --test_output=errors ^ |
|
||||||
--verbose_failures ^ |
|
||||||
--invocation_id=%BAZEL_INTERNAL_INVOCATION_ID% ^ |
|
||||||
--google_credentials=%KOKORO_BAZEL_AUTH_CREDENTIAL% ^ |
|
||||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/%KOKORO_JOB_NAME% |
|
||||||
|
|
||||||
@rem Regenerate stale CMake configs. |
|
||||||
@rem LINT.IfChange(staleness_tests) |
|
||||||
bazel test //src:cmake_lists_staleness_test || call bazel-bin\src\cmake_lists_staleness_test.exe --fix |
|
||||||
bazel test //src/google/protobuf:well_known_types_staleness_test || call bazel-bin\src\google\protobuf\well_known_types_staleness_test.exe --fix |
|
||||||
@rem LINT.ThenChange(//depot/google3/third_party/protobuf/github/regenerate_stale_files.sh) |
|
||||||
Loading…
Reference in new issue