From 07a75427bd1447535708a3e33f26d31bca18074b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Apr 2022 11:31:41 +0200 Subject: [PATCH] Simplify running bazel with structured test results on CI (#29353) * add bazel_report_helper.py * simplify bazel invocations in selected builds * introduce __main__ * update gitignore * introduce sleep constant * add type annotations * use f-strings * Revert "use f-strings" This reverts commit f970d6a40bd134cada2f01d8bac1c224138cb35f. --- .gitignore | 6 + .../linux/grpc_bazel_on_foundry_base.sh | 26 +- .../linux/grpc_python_bazel_test_in_docker.sh | 19 +- .../macos/grpc_run_bazel_c_cpp_tests.sh | 28 +- tools/internal_ci/windows/bazel_rbe.bat | 15 +- .../python_utils/bazel_report_helper.py | 258 ++++++++++++++++++ 6 files changed, 286 insertions(+), 66 deletions(-) create mode 100755 tools/run_tests/python_utils/bazel_report_helper.py diff --git a/.gitignore b/.gitignore index f236d45210b..4e1aedf5d76 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ Gemfile.lock # Temporary test reports report.xml */sponge_log.xml +*/success_log_to_rename.xml latency_trace.txt latency_trace.*.txt @@ -122,6 +123,11 @@ bazel-* bazel_format_virtual_environment/ tools/bazel-* +# Bazel wrapper +bazel_wrapper +bazel_wrapper.bat +bazel_wrapper.bazelrc + # Debug output gdb.txt diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index d1c7e924899..deaf68f0600 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -23,28 +23,10 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc # make sure bazel is available tools/bazel version -# to get "bazel" link for kokoro build, we need to generate -# invocation UUID, set a flag for bazel to use it -# and upload "bazel_invocation_ids" file as artifact. -BAZEL_INVOCATION_ID="$(uuidgen)" -echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_rbe -tools/bazel \ +bazel_rbe/bazel_wrapper \ --bazelrc=tools/remote_build/linux_kokoro.bazelrc \ test \ - --invocation_id="${BAZEL_INVOCATION_ID}" \ - --workspace_status_command=tools/remote_build/workspace_status_kokoro.sh \ - $@ \ - -- //test/... || FAILED="true" - -if [ "$UPLOAD_TEST_RESULTS" != "" ] -then - # Sleep to let ResultStore finish writing results before querying - sleep 60 - python3 ./tools/run_tests/python_utils/upload_rbe_results.py -fi - -if [ "$FAILED" != "" ] -then - exit 1 -fi + "$@" \ + -- //test/... diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index e30516f5747..a36cd416a96 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -14,18 +14,23 @@ # limitations under the License. # # Test full Bazel -# -# NOTE: No empty lines should appear in this file before igncr is set! -set -ex -o igncr || set -ex + +set -ex mkdir -p /var/local/git git clone /var/local/jenkins/grpc /var/local/git/grpc (cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') -cd /var/local/git/grpc/test +cd /var/local/git/grpc TEST_TARGETS="//src/python/... //tools/distrib/python/grpcio_tools/... //examples/python/..." BAZEL_FLAGS="--test_output=errors" -bazel test ${BAZEL_FLAGS} ${TEST_TARGETS} -bazel test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS} -bazel test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS} + +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests +python_bazel_tests/bazel_wrapper test ${BAZEL_FLAGS} ${TEST_TARGETS} + +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_single_threaded_unary_streams +python_bazel_tests_single_threaded_unary_streams/bazel_wrapper test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS} + +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_poller_engine +python_bazel_tests_poller_engine/bazel_wrapper test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS} diff --git a/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh b/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh index 6dd2cfd7a5b..52e2d133f4a 100644 --- a/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh +++ b/tools/internal_ci/macos/grpc_run_bazel_c_cpp_tests.sh @@ -28,14 +28,6 @@ tools/bazel version ./tools/run_tests/start_port_server.py -# to get "bazel" link for kokoro build, we need to generate -# invocation UUID, set a flag for bazel to use it -# and upload "bazel_invocation_ids" file as artifact. -# NOTE: UUID needs to be in lowercase for the result link to work -# (on mac "uuidgen" outputs uppercase UUID) -BAZEL_INVOCATION_ID="$(uuidgen | tr '[:upper:]' '[:lower:]')" -echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" - # for kokoro mac workers, exact image version is store in a well-known location on disk KOKORO_IMAGE_VERSION="$(cat /VERSION)" @@ -48,25 +40,13 @@ BAZEL_REMOTE_CACHE_ARGS=( --remote_default_exec_properties="grpc_cache_silo_key2=${KOKORO_IMAGE_VERSION}" ) +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_c_cpp_tests + # run all C/C++ tests -tools/bazel \ +bazel_c_cpp_tests/bazel_wrapper \ --bazelrc=tools/remote_build/mac.bazelrc \ test \ - --invocation_id="${BAZEL_INVOCATION_ID}" \ - --workspace_status_command=tools/remote_build/workspace_status_kokoro.sh \ --google_credentials="${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json" \ "${BAZEL_REMOTE_CACHE_ARGS[@]}" \ $BAZEL_FLAGS \ - -- //test/... || FAILED="true" - -if [ "$UPLOAD_TEST_RESULTS" != "" ] -then - # Sleep to let ResultStore finish writing results before querying - sleep 60 - PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py -fi - -if [ "$FAILED" != "" ] -then - exit 1 -fi + -- //test/... diff --git a/tools/internal_ci/windows/bazel_rbe.bat b/tools/internal_ci/windows/bazel_rbe.bat index 7a7834759f0..4bd9cb6c9ee 100644 --- a/tools/internal_ci/windows/bazel_rbe.bat +++ b/tools/internal_ci/windows/bazel_rbe.bat @@ -33,17 +33,6 @@ bash -c "tools/bazel --version && cp tools/bazel-*.exe /c/bazel/bazel.exe" set PATH=C:\bazel;%PATH% bazel --version -@rem Generate a random UUID and store in "bazel_invocation_ids" artifact file -powershell -Command "[guid]::NewGuid().ToString()" >%KOKORO_ARTIFACTS_DIR%/bazel_invocation_ids -set /p BAZEL_INVOCATION_ID=<%KOKORO_ARTIFACTS_DIR%/bazel_invocation_ids +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_rbe -bazel --bazelrc=tools/remote_build/windows.bazelrc --output_user_root=T:\_bazel_output test --invocation_id="%BAZEL_INVOCATION_ID%" %BAZEL_FLAGS% --workspace_status_command=tools/remote_build/workspace_status_kokoro.bat //test/... -set BAZEL_EXITCODE=%errorlevel% - -if not "%UPLOAD_TEST_RESULTS%"=="" ( - @rem Sleep to let ResultStore finish writing results before querying - sleep 60 - python3 tools/run_tests/python_utils/upload_rbe_results.py || exit /b 1 -) - -exit /b %BAZEL_EXITCODE% +call bazel_rbe/bazel_wrapper.bat --bazelrc=tools/remote_build/windows.bazelrc --output_user_root=T:\_bazel_output test %BAZEL_FLAGS% -- //test/... || exit /b 1 diff --git a/tools/run_tests/python_utils/bazel_report_helper.py b/tools/run_tests/python_utils/bazel_report_helper.py new file mode 100755 index 00000000000..c62807a5856 --- /dev/null +++ b/tools/run_tests/python_utils/bazel_report_helper.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Helps with running bazel with extra settings to generate structured test reports in CI.""" + +import argparse +import os +import platform +import sys +import uuid + +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) +os.chdir(_ROOT) + +# How long to sleep before querying Resultstore API and uploading to bigquery +# (to let ResultStore finish writing results from the bazel invocation that has +# just finished). +_UPLOAD_RBE_RESULTS_DELAY_SECONDS = 60 + + +def _platform_string(): + """Detect current platform""" + if platform.system() == 'Windows': + return 'windows' + elif platform.system()[:7] == 'MSYS_NT': + return 'windows' + elif platform.system() == 'Darwin': + return 'mac' + elif platform.system() == 'Linux': + return 'linux' + else: + return 'posix' + + +def _append_to_kokoro_bazel_invocations(invocation_id: str) -> None: + """Kokoro can display "Bazel" result link on kokoro jobs if told so.""" + # to get "bazel" link for kokoro build, we need to upload + # the "bazel_invocation_ids" file with bazel invocation ID as artifact. + kokoro_artifacts_dir = os.getenv('KOKORO_ARTIFACTS_DIR') + if kokoro_artifacts_dir: + # append the bazel invocation UUID to the bazel_invocation_ids file. + with open(os.path.join(kokoro_artifacts_dir, 'bazel_invocation_ids'), + 'a') as f: + f.write(invocation_id + '\n') + print( + 'Added invocation ID %s to kokoro "bazel_invocation_ids" artifact' % + invocation_id, + file=sys.stderr) + else: + print( + 'Skipped adding invocation ID %s to kokoro "bazel_invocation_ids" artifact' + % invocation_id, + file=sys.stderr) + pass + + +def _generate_junit_report_string(report_suite_name: str, invocation_id: str, + success: bool) -> None: + """Generate sponge_log.xml formatted report, that will make the bazel invocation reachable as a target in resultstore UI / sponge.""" + bazel_invocation_url = 'https://source.cloud.google.com/results/invocations/%s' % invocation_id + package_name = report_suite_name + # set testcase name to invocation URL. That way, the link will be displayed in some form + # resultstore UI and sponge even in case the bazel invocation succeeds. + testcase_name = bazel_invocation_url + if success: + # unfortunately, neither resultstore UI nor sponge display the "system-err" output (or any other tags) + # on a passing test case. But at least we tried. + test_output_tag = 'PASSED. See invocation results here: %s' % bazel_invocation_url + else: + # The failure output will be displayes in both resultstore UI and sponge when clicking on the failing testcase. + test_output_tag = 'FAILED. See bazel invocation results here: %s' % bazel_invocation_url + + lines = [ + '', + '' % + (report_suite_name, package_name), + '' % testcase_name, + test_output_tag, + '' + '', + '', + ] + return '\n'.join(lines) + + +def _create_bazel_wrapper(report_path: str, report_suite_name: str, + invocation_id: str, upload_results: bool) -> None: + """Create a "bazel wrapper" script that will execute bazel with extra settings and postprocessing.""" + + os.makedirs(report_path, exist_ok=True) + + bazel_wrapper_filename = os.path.join(report_path, 'bazel_wrapper') + bazel_wrapper_bat_filename = bazel_wrapper_filename + '.bat' + bazel_rc_filename = os.path.join(report_path, 'bazel_wrapper.bazelrc') + + # put xml reports in a separate directory if requested by GRPC_TEST_REPORT_BASE_DIR + report_base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None) + xml_report_path = os.path.abspath( + os.path.join(report_base_dir, report_path + ) if report_base_dir else report_path) + os.makedirs(xml_report_path, exist_ok=True) + + failing_report_filename = os.path.join(xml_report_path, 'sponge_log.xml') + success_report_filename = os.path.join(xml_report_path, + 'success_log_to_rename.xml') + + if _platform_string() == 'windows': + workspace_status_command = 'tools/remote_build/workspace_status_kokoro.bat' + else: + workspace_status_command = 'tools/remote_build/workspace_status_kokoro.sh' + + # generate RC file with the bazel flags we want to use apply. + # Using an RC file solves problems with flag ordering in the wrapper. + # (e.g. some flags need to come after the build/test command) + with open(bazel_rc_filename, 'w') as f: + f.write('build --invocation_id="%s"\n' % invocation_id) + f.write('build --workspace_status_command="%s"\n' % + workspace_status_command) + + # generate "failing" and "success" report + # the "failing" is named as "sponge_log.xml", which is the name picked up by sponge/resultstore + # so the failing report will be used by default (unless we later replace the report with + # one that says "success"). That way if something goes wrong before bazel is run, + # there will at least be a "failing" target that indicates that (we really don't want silent failures). + with open(failing_report_filename, 'w') as f: + f.write( + _generate_junit_report_string(report_suite_name, + invocation_id, + success=False)) + with open(success_report_filename, 'w') as f: + f.write( + _generate_junit_report_string(report_suite_name, + invocation_id, + success=True)) + + # generate the bazel wrapper for linux/macos + with open(bazel_wrapper_filename, 'w') as f: + intro_lines = [ + '#!/bin/bash', + 'set -ex', + '', + 'tools/bazel --bazelrc="%s" "$@" || FAILED=true' % + bazel_rc_filename, + '', + ] + + if upload_results: + upload_results_lines = [ + 'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, + 'PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s"' + % invocation_id, + '', + ] + else: + upload_results_lines = [] + + outro_lines = [ + 'if [ "$FAILED" != "" ]', + 'then', + ' exit 1', + 'else', + ' # success: plant the pre-generated xml report that says "success"', + ' mv -f %s %s' % + (success_report_filename, failing_report_filename), + 'fi', + ] + + lines = [ + line + '\n' + for line in intro_lines + upload_results_lines + outro_lines + ] + f.writelines(lines) + os.chmod(bazel_wrapper_filename, 0o775) # make the unix wrapper executable + + # generate bazel wrapper for windows + with open(bazel_wrapper_bat_filename, 'w') as f: + intro_lines = [ + '@echo on', + '', + 'bazel --bazelrc="%s" %%*' % bazel_rc_filename, + 'set BAZEL_EXITCODE=%errorlevel%', + '', + ] + + if upload_results: + upload_results_lines = [ + 'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, + 'python3 tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s" || exit /b 1' + % invocation_id, + '', + ] + else: + upload_results_lines = [] + + outro_lines = [ + 'if %BAZEL_EXITCODE% == 0 (', + ' @rem success: plant the pre-generated xml report that says "success"', + ' mv -f %s %s' % + (success_report_filename, failing_report_filename), + ')', + 'exit /b %BAZEL_EXITCODE%', + ] + + lines = [ + line + '\n' + for line in intro_lines + upload_results_lines + outro_lines + ] + f.writelines(lines) + + print('Bazel invocation ID: %s' % invocation_id, file=sys.stderr) + print('Upload test results to BigQuery after bazel runs: %s' % + upload_results, + file=sys.stderr) + print('Generated bazel wrapper: %s' % bazel_wrapper_filename, + file=sys.stderr) + print('Generated bazel wrapper: %s' % bazel_wrapper_bat_filename, + file=sys.stderr) + + +if __name__ == '__main__': + # parse command line + argp = argparse.ArgumentParser( + description= + 'Generate bazel wrapper to help with bazel test reports in CI.') + argp.add_argument( + '--report_path', + required=True, + type=str, + help= + 'Path under which the bazel wrapper and other files are going to be generated' + ) + argp.add_argument('--report_suite_name', + default='bazel_invocations', + type=str, + help='Test suite name to use in generated XML report') + args = argp.parse_args() + + # generate new bazel invocation ID + invocation_id = str(uuid.uuid4()) + + report_path = args.report_path + report_suite_name = args.report_suite_name + upload_results = True if os.getenv('UPLOAD_TEST_RESULTS') else False + + _append_to_kokoro_bazel_invocations(invocation_id) + _create_bazel_wrapper(report_path, report_suite_name, invocation_id, + upload_results)