diff --git a/.gitignore b/.gitignore
index f236d45210b..4e1aedf5d76 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,6 +63,7 @@ Gemfile.lock
# Temporary test reports
report.xml
*/sponge_log.xml
+*/success_log_to_rename.xml
latency_trace.txt
latency_trace.*.txt
@@ -122,6 +123,11 @@ bazel-*
bazel_format_virtual_environment/
tools/bazel-*
+# Bazel wrapper
+bazel_wrapper
+bazel_wrapper.bat
+bazel_wrapper.bazelrc
+
# Debug output
gdb.txt
diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
index d1c7e924899..deaf68f0600 100755
--- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
+++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh
@@ -23,28 +23,10 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc
# make sure bazel is available
tools/bazel version
-# to get "bazel" link for kokoro build, we need to generate
-# invocation UUID, set a flag for bazel to use it
-# and upload "bazel_invocation_ids" file as artifact.
-BAZEL_INVOCATION_ID="$(uuidgen)"
-echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids"
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_rbe
-tools/bazel \
+bazel_rbe/bazel_wrapper \
--bazelrc=tools/remote_build/linux_kokoro.bazelrc \
test \
- --invocation_id="${BAZEL_INVOCATION_ID}" \
- --workspace_status_command=tools/remote_build/workspace_status_kokoro.sh \
- $@ \
- -- //test/... || FAILED="true"
-
-if [ "$UPLOAD_TEST_RESULTS" != "" ]
-then
- # Sleep to let ResultStore finish writing results before querying
- sleep 60
- python3 ./tools/run_tests/python_utils/upload_rbe_results.py
-fi
-
-if [ "$FAILED" != "" ]
-then
- exit 1
-fi
+ "$@" \
+ -- //test/...
diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh
index e30516f5747..a36cd416a96 100755
--- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh
+++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh
@@ -14,18 +14,23 @@
# limitations under the License.
#
# Test full Bazel
-#
-# NOTE: No empty lines should appear in this file before igncr is set!
-set -ex -o igncr || set -ex
+
+set -ex
mkdir -p /var/local/git
git clone /var/local/jenkins/grpc /var/local/git/grpc
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
${name}')
-cd /var/local/git/grpc/test
+cd /var/local/git/grpc
TEST_TARGETS="//src/python/... //tools/distrib/python/grpcio_tools/... //examples/python/..."
BAZEL_FLAGS="--test_output=errors"
-bazel test ${BAZEL_FLAGS} ${TEST_TARGETS}
-bazel test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS}
-bazel test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS}
+
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests
+python_bazel_tests/bazel_wrapper test ${BAZEL_FLAGS} ${TEST_TARGETS}
+
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_single_threaded_unary_streams
+python_bazel_tests_single_threaded_unary_streams/bazel_wrapper test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS}
+
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_poller_engine
+python_bazel_tests_poller_engine/bazel_wrapper test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS}
diff --git a/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh b/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh
index 6dd2cfd7a5b..52e2d133f4a 100644
--- a/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh
+++ b/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh
@@ -28,14 +28,6 @@ tools/bazel version
./tools/run_tests/start_port_server.py
-# to get "bazel" link for kokoro build, we need to generate
-# invocation UUID, set a flag for bazel to use it
-# and upload "bazel_invocation_ids" file as artifact.
-# NOTE: UUID needs to be in lowercase for the result link to work
-# (on mac "uuidgen" outputs uppercase UUID)
-BAZEL_INVOCATION_ID="$(uuidgen | tr '[:upper:]' '[:lower:]')"
-echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids"
-
# for kokoro mac workers, exact image version is store in a well-known location on disk
KOKORO_IMAGE_VERSION="$(cat /VERSION)"
@@ -48,25 +40,13 @@ BAZEL_REMOTE_CACHE_ARGS=(
--remote_default_exec_properties="grpc_cache_silo_key2=${KOKORO_IMAGE_VERSION}"
)
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_c_cpp_tests
+
# run all C/C++ tests
-tools/bazel \
+bazel_c_cpp_tests/bazel_wrapper \
--bazelrc=tools/remote_build/mac.bazelrc \
test \
- --invocation_id="${BAZEL_INVOCATION_ID}" \
- --workspace_status_command=tools/remote_build/workspace_status_kokoro.sh \
--google_credentials="${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json" \
"${BAZEL_REMOTE_CACHE_ARGS[@]}" \
$BAZEL_FLAGS \
- -- //test/... || FAILED="true"
-
-if [ "$UPLOAD_TEST_RESULTS" != "" ]
-then
- # Sleep to let ResultStore finish writing results before querying
- sleep 60
- PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py
-fi
-
-if [ "$FAILED" != "" ]
-then
- exit 1
-fi
+ -- //test/...
diff --git a/tools/internal_ci/windows/bazel_rbe.bat b/tools/internal_ci/windows/bazel_rbe.bat
index 7a7834759f0..4bd9cb6c9ee 100644
--- a/tools/internal_ci/windows/bazel_rbe.bat
+++ b/tools/internal_ci/windows/bazel_rbe.bat
@@ -33,17 +33,6 @@ bash -c "tools/bazel --version && cp tools/bazel-*.exe /c/bazel/bazel.exe"
set PATH=C:\bazel;%PATH%
bazel --version
-@rem Generate a random UUID and store in "bazel_invocation_ids" artifact file
-powershell -Command "[guid]::NewGuid().ToString()" >%KOKORO_ARTIFACTS_DIR%/bazel_invocation_ids
-set /p BAZEL_INVOCATION_ID=<%KOKORO_ARTIFACTS_DIR%/bazel_invocation_ids
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_rbe
-bazel --bazelrc=tools/remote_build/windows.bazelrc --output_user_root=T:\_bazel_output test --invocation_id="%BAZEL_INVOCATION_ID%" %BAZEL_FLAGS% --workspace_status_command=tools/remote_build/workspace_status_kokoro.bat //test/...
-set BAZEL_EXITCODE=%errorlevel%
-
-if not "%UPLOAD_TEST_RESULTS%"=="" (
- @rem Sleep to let ResultStore finish writing results before querying
- sleep 60
- python3 tools/run_tests/python_utils/upload_rbe_results.py || exit /b 1
-)
-
-exit /b %BAZEL_EXITCODE%
+call bazel_rbe/bazel_wrapper.bat --bazelrc=tools/remote_build/windows.bazelrc --output_user_root=T:\_bazel_output test %BAZEL_FLAGS% -- //test/... || exit /b 1
diff --git a/tools/run_tests/python_utils/bazel_report_helper.py b/tools/run_tests/python_utils/bazel_report_helper.py
new file mode 100755
index 00000000000..c62807a5856
--- /dev/null
+++ b/tools/run_tests/python_utils/bazel_report_helper.py
@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helps with running bazel with extra settings to generate structured test reports in CI."""
+
+import argparse
+import os
+import platform
+import sys
+import uuid
+
+_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
+os.chdir(_ROOT)
+
+# How long to sleep before querying Resultstore API and uploading to bigquery
+# (to let ResultStore finish writing results from the bazel invocation that has
+# just finished).
+_UPLOAD_RBE_RESULTS_DELAY_SECONDS = 60
+
+
+def _platform_string():
+ """Detect current platform"""
+ if platform.system() == 'Windows':
+ return 'windows'
+ elif platform.system()[:7] == 'MSYS_NT':
+ return 'windows'
+ elif platform.system() == 'Darwin':
+ return 'mac'
+ elif platform.system() == 'Linux':
+ return 'linux'
+ else:
+ return 'posix'
+
+
+def _append_to_kokoro_bazel_invocations(invocation_id: str) -> None:
+ """Kokoro can display "Bazel" result link on kokoro jobs if told so."""
+ # to get "bazel" link for kokoro build, we need to upload
+ # the "bazel_invocation_ids" file with bazel invocation ID as artifact.
+ kokoro_artifacts_dir = os.getenv('KOKORO_ARTIFACTS_DIR')
+ if kokoro_artifacts_dir:
+ # append the bazel invocation UUID to the bazel_invocation_ids file.
+ with open(os.path.join(kokoro_artifacts_dir, 'bazel_invocation_ids'),
+ 'a') as f:
+ f.write(invocation_id + '\n')
+ print(
+ 'Added invocation ID %s to kokoro "bazel_invocation_ids" artifact' %
+ invocation_id,
+ file=sys.stderr)
+ else:
+ print(
+ 'Skipped adding invocation ID %s to kokoro "bazel_invocation_ids" artifact'
+ % invocation_id,
+ file=sys.stderr)
+ pass
+
+
+def _generate_junit_report_string(report_suite_name: str, invocation_id: str,
+ success: bool) -> None:
+ """Generate sponge_log.xml formatted report, that will make the bazel invocation reachable as a target in resultstore UI / sponge."""
+ bazel_invocation_url = 'https://source.cloud.google.com/results/invocations/%s' % invocation_id
+ package_name = report_suite_name
+ # set testcase name to invocation URL. That way, the link will be displayed in some form
+ # resultstore UI and sponge even in case the bazel invocation succeeds.
+ testcase_name = bazel_invocation_url
+ if success:
+ # unfortunately, neither resultstore UI nor sponge display the "system-err" output (or any other tags)
+ # on a passing test case. But at least we tried.
+ test_output_tag = '<system-err>PASSED. See invocation results here: %s</system-err>' % bazel_invocation_url
+ else:
+ # The failure output will be displayes in both resultstore UI and sponge when clicking on the failing testcase.
+ test_output_tag = '<failure message="Failure">FAILED. See bazel invocation results here: %s</failure>' % bazel_invocation_url
+
+ lines = [
+ '<testsuites>',
+ '<testsuite id="1" name="%s" package="%s">' %
+ (report_suite_name, package_name),
+ '<testcase name="%s">' % testcase_name,
+ test_output_tag,
+ '</testcase>'
+ '</testsuite>',
+ '</testsuites>',
+ ]
+ return '\n'.join(lines)
+
+
+def _create_bazel_wrapper(report_path: str, report_suite_name: str,
+ invocation_id: str, upload_results: bool) -> None:
+ """Create a "bazel wrapper" script that will execute bazel with extra settings and postprocessing."""
+
+ os.makedirs(report_path, exist_ok=True)
+
+ bazel_wrapper_filename = os.path.join(report_path, 'bazel_wrapper')
+ bazel_wrapper_bat_filename = bazel_wrapper_filename + '.bat'
+ bazel_rc_filename = os.path.join(report_path, 'bazel_wrapper.bazelrc')
+
+ # put xml reports in a separate directory if requested by GRPC_TEST_REPORT_BASE_DIR
+ report_base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None)
+ xml_report_path = os.path.abspath(
+ os.path.join(report_base_dir, report_path
+ ) if report_base_dir else report_path)
+ os.makedirs(xml_report_path, exist_ok=True)
+
+ failing_report_filename = os.path.join(xml_report_path, 'sponge_log.xml')
+ success_report_filename = os.path.join(xml_report_path,
+ 'success_log_to_rename.xml')
+
+ if _platform_string() == 'windows':
+ workspace_status_command = 'tools/remote_build/workspace_status_kokoro.bat'
+ else:
+ workspace_status_command = 'tools/remote_build/workspace_status_kokoro.sh'
+
+ # generate RC file with the bazel flags we want to use apply.
+ # Using an RC file solves problems with flag ordering in the wrapper.
+ # (e.g. some flags need to come after the build/test command)
+ with open(bazel_rc_filename, 'w') as f:
+ f.write('build --invocation_id="%s"\n' % invocation_id)
+ f.write('build --workspace_status_command="%s"\n' %
+ workspace_status_command)
+
+ # generate "failing" and "success" report
+ # the "failing" is named as "sponge_log.xml", which is the name picked up by sponge/resultstore
+ # so the failing report will be used by default (unless we later replace the report with
+ # one that says "success"). That way if something goes wrong before bazel is run,
+ # there will at least be a "failing" target that indicates that (we really don't want silent failures).
+ with open(failing_report_filename, 'w') as f:
+ f.write(
+ _generate_junit_report_string(report_suite_name,
+ invocation_id,
+ success=False))
+ with open(success_report_filename, 'w') as f:
+ f.write(
+ _generate_junit_report_string(report_suite_name,
+ invocation_id,
+ success=True))
+
+ # generate the bazel wrapper for linux/macos
+ with open(bazel_wrapper_filename, 'w') as f:
+ intro_lines = [
+ '#!/bin/bash',
+ 'set -ex',
+ '',
+ 'tools/bazel --bazelrc="%s" "$@" || FAILED=true' %
+ bazel_rc_filename,
+ '',
+ ]
+
+ if upload_results:
+ upload_results_lines = [
+ 'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS,
+ 'PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s"'
+ % invocation_id,
+ '',
+ ]
+ else:
+ upload_results_lines = []
+
+ outro_lines = [
+ 'if [ "$FAILED" != "" ]',
+ 'then',
+ ' exit 1',
+ 'else',
+ ' # success: plant the pre-generated xml report that says "success"',
+ ' mv -f %s %s' %
+ (success_report_filename, failing_report_filename),
+ 'fi',
+ ]
+
+ lines = [
+ line + '\n'
+ for line in intro_lines + upload_results_lines + outro_lines
+ ]
+ f.writelines(lines)
+ os.chmod(bazel_wrapper_filename, 0o775) # make the unix wrapper executable
+
+ # generate bazel wrapper for windows
+ with open(bazel_wrapper_bat_filename, 'w') as f:
+ intro_lines = [
+ '@echo on',
+ '',
+ 'bazel --bazelrc="%s" %%*' % bazel_rc_filename,
+ 'set BAZEL_EXITCODE=%errorlevel%',
+ '',
+ ]
+
+ if upload_results:
+ upload_results_lines = [
+ 'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS,
+ 'python3 tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s" || exit /b 1'
+ % invocation_id,
+ '',
+ ]
+ else:
+ upload_results_lines = []
+
+ outro_lines = [
+ 'if %BAZEL_EXITCODE% == 0 (',
+ ' @rem success: plant the pre-generated xml report that says "success"',
+ ' mv -f %s %s' %
+ (success_report_filename, failing_report_filename),
+ ')',
+ 'exit /b %BAZEL_EXITCODE%',
+ ]
+
+ lines = [
+ line + '\n'
+ for line in intro_lines + upload_results_lines + outro_lines
+ ]
+ f.writelines(lines)
+
+ print('Bazel invocation ID: %s' % invocation_id, file=sys.stderr)
+ print('Upload test results to BigQuery after bazel runs: %s' %
+ upload_results,
+ file=sys.stderr)
+ print('Generated bazel wrapper: %s' % bazel_wrapper_filename,
+ file=sys.stderr)
+ print('Generated bazel wrapper: %s' % bazel_wrapper_bat_filename,
+ file=sys.stderr)
+
+
+if __name__ == '__main__':
+ # parse command line
+ argp = argparse.ArgumentParser(
+ description=
+ 'Generate bazel wrapper to help with bazel test reports in CI.')
+ argp.add_argument(
+ '--report_path',
+ required=True,
+ type=str,
+ help=
+ 'Path under which the bazel wrapper and other files are going to be generated'
+ )
+ argp.add_argument('--report_suite_name',
+ default='bazel_invocations',
+ type=str,
+ help='Test suite name to use in generated XML report')
+ args = argp.parse_args()
+
+ # generate new bazel invocation ID
+ invocation_id = str(uuid.uuid4())
+
+ report_path = args.report_path
+ report_suite_name = args.report_suite_name
+ upload_results = True if os.getenv('UPLOAD_TEST_RESULTS') else False
+
+ _append_to_kokoro_bazel_invocations(invocation_id)
+ _create_bazel_wrapper(report_path, report_suite_name, invocation_id,
+ upload_results)