[Kokoro] Emit XML test logs on macos-next. (#9709)

This enables googletest XML output on the macos-next builders, and adds logic to collect the results.

The log collection logic is slightly complex, but it should be reusable in other contexts. The idea is to capture stdout/stderr for build steps along with googletest XML reports into a temporary directory, then stage those into paths expected for artifacts.

kokoro/caplog.sh

@@ -0,0 +1,92 @@
+# Log capturing for the Kokoro runtime environment.
+#
+# This script should be `source`d from Kokoro build scripts to configure log
+# capturing when running under Kokoro.
+#
+# When not running under Kokoro, no logs will be collected. If you want to run
+# locally and collect logs anyway, set the KOKORO_ARTIFACTS_DIR environment
+# variable to a directory where the logs should go.
+#
+# The job `.cfg` file needs the following stanzas to declare the captured logs
+# as outputs (yes, these are globs, not regexes):
+#
+#   action: {
+#     define_artifacts: {
+#       regex: "**/*sponge_log.log"
+#       regex: "**/*sponge_log.xml"
+#     }
+#   }
+#
+# Use the provided functions below as build/test fixtures, e.g.:
+#
+#   source kokoro/capture_logs.sh
+#   caplog build/step1 <build command>
+#   caplog tests/step2 <test command>
+#
+# If log capturing is enabled, this script will set some variables that can be
+# used if necessary:
+#
+#   CAPLOG_DIR         is used for logs
+#   CAPLOG_CMAKE_ARGS  contains extra cmake args to enable test XML output
+#   CAPLOG_CTEST_ARGS  contains extra ctest args to capture combined test logs
+#
+# For example:
+#
+#   if [[ -v CAPLOG_DIR_BUILD ]]; then
+#     cp extra_diagnostics.log "${CAPLOG_DIR_BUILD}/diagnostics.log"
+#   fi
+#
+#   # Use ${...:-} form under `set -u`:
+#   caplog build/01_configure cmake -G Ninja ${CAPLOG_CMAKE_ARGS:-}
+#   caplog build/02_build     cmake --build
+#   caplog test/03_test       ctest ${CAPLOG_CTEST_ARGS:-}
+
+if [[ -z ${KOKORO_ARTIFACTS_DIR:-} ]]; then
+  function caplog() { shift; "$@"; }
+else
+
+  CAPLOG_DIR="$(mktemp -d)"
+  CAPLOG_CMAKE_ARGS="-Dprotobuf_TEST_XML_OUTDIR=${CAPLOG_DIR}/tests/"
+  CAPLOG_CTEST_ARGS="--verbose"
+
+  # Captures the stdout/stderr of a command to a named log file.
+  # Usage: caplog NAME COMMAND [ARGS...]
+  function caplog() {
+    _name="${CAPLOG_DIR}/${1%.log}.log"; shift
+    mkdir -p "${_name%/*}"
+    date
+    time ( "$@" 2>&1 | tee "${_name}" )
+    if [[ $? != 0 ]] ; then
+      cat "${_name}"
+      return 1
+    fi
+  }
+
+  # Trap handler: renames logs on script exit so they will be found by Kokoro.
+  function _caplog_onexit() {
+    _rc=$?
+    set +x
+    echo "Collecting logs [${BASH_SOURCE}]"
+
+    find "${CAPLOG_DIR}" -type f -name '*.log' \
+      | while read _textlog; do
+      # Ensure an XML file exists for each .log file.
+      touch ${_textlog%.log}.xml
+    done
+
+    find "${CAPLOG_DIR}" -type f \( -name '*.xml' -or -name '*.log' \) \
+      | while read _src; do
+      # Move to artifacts dir, preserving the path relative to CAPLOG_DIR.
+      # The filename changes from foo/bar.log to foo/bar/sponge_log.log.
+      _logfile=${_src/${CAPLOG_DIR}\//}
+      _stem=${KOKORO_ARTIFACTS_DIR}/${_logfile%.*}
+      _ext=${_logfile##*.}
+      mkdir -p ${_stem}
+      mv -v "${_src}" "${_stem}/sponge_log.${_ext}"
+    done
+    rm -rv "${CAPLOG_DIR}"
+    exit ${_rc}
+  }
+  trap _caplog_onexit EXIT
+
+fi

kokoro/macos-next/cpp/build.sh

@@ -1,51 +1,55 @@
-#!/bin/bash -ex -o pipefail
+#!/bin/bash -eux
 #
 # Build file to set up and run tests
 
-#
-# Set up logging output location
-#
-: ${KOKORO_ARTIFACTS_DIR:=/tmp/protobuf_test_logs}
-: ${BUILD_LOGDIR:=$KOKORO_ARTIFACTS_DIR/logs}
-mkdir -p ${BUILD_LOGDIR}
+set -o pipefail
 
-
-#
-# Change to repo root
-#
 if [[ -h /tmpfs ]] && [[ ${PWD} == /tmpfs/src ]]; then
   # Workaround for internal Kokoro bug: b/227401944
-  cd /Volumes/BuildData/tmpfs/src/github/protobuf
-else
-  cd $(dirname $0)/../../..
+  cd /Volumes/BuildData/tmpfs/src
 fi
 
+# These vars can be changed when running manually, e.g.:
+#
+#   % BUILD_CONFIG=RelWithDebInfo path/to/build.sh
+
+# By default, build using Debug config.
+: ${BUILD_CONFIG:=Debug}
+
+# By default, find the sources based on this script path.
+: ${SOURCE_DIR:=$(cd $(dirname $0)/../../..; pwd)}
+
+# By default, put outputs under <git root>/cmake/build.
+: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build}
+
+source ${SOURCE_DIR}/kokoro/caplog.sh
+
 #
 # Update submodules
 #
-git submodule update --init --recursive
+git -C "${SOURCE_DIR}" submodule update --init --recursive
 
 #
 # Configure and build in a separate directory
 #
-mkdir -p cmake/build
-cd cmake/build
+mkdir -p "${BUILD_DIR}"
 
-cmake -G Xcode ../.. \
-  2>&1 | tee ${BUILD_LOGDIR}/01_configure.log
+caplog 01_configure \
+  cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" ${CAPLOG_CMAKE_ARGS:-}
 
-cp CMakeFiles/CMake*.log ${BUILD_LOGDIR}
+if [[ -n ${CAPLOG_DIR:-} ]]; then
+  mkdir -p "${CAPLOG_DIR}/CMakeFiles"
+  cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles"
+fi
 
-cmake --build . --config Debug \
-  2>&1 | tee ${BUILD_LOGDIR}/02_build.log
+caplog 02_build \
+  cmake --build "${BUILD_DIR}" --config "${BUILD_CONFIG}"
 
 #
 # Run tests
 #
-ctest -C Debug --verbose --quiet \
-  --output-log ${BUILD_LOGDIR}/03_test.log
-
-#
-# Compress outputs
-#
-gzip ${BUILD_LOGDIR}/*.log
+(
+  cd "${BUILD_DIR}"
+  caplog 03_combined_testlog \
+    ctest -C "${BUILD_CONFIG}" -j4 ${CAPLOG_CTEST_ARGS:-}
+)

kokoro/macos-next/cpp/continuous.cfg

@@ -9,7 +9,5 @@ action: {
   define_artifacts: {
     regex: "**/*sponge_log.log"
     regex: "**/*sponge_log.xml"
-    regex: "logs/*.log"
-    regex: "logs/*.log.gz"
   }
 }

kokoro/macos-next/cpp/presubmit.cfg

@@ -9,7 +9,5 @@ action: {
   define_artifacts: {
     regex: "**/*sponge_log.log"
     regex: "**/*sponge_log.xml"
-    regex: "logs/*.log"
-    regex: "logs/*.log.gz"
   }
 }