mirror of https://github.com/grpc/grpc.git
Simplify running bazel with structured test results on CI (#29353)
* add bazel_report_helper.py
* simplify bazel invocations in selected builds
* introduce __main__
* update gitignore
* introduce sleep constant
* add type annotations
* use f-strings
* Revert "use f-strings"
This reverts commit f970d6a40b.
pull/29376/head
Jan Tattermusch
3 years ago
committed by
GitHub
parent
5850cba295
commit
07a75427bd
6 changed files with 286 additions and 66 deletions
@ -0,0 +1,258 @@ |
||||
#!/usr/bin/env python3 |
||||
# Copyright 2022 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Helps with running bazel with extra settings to generate structured test reports in CI.""" |
||||
|
||||
import argparse |
||||
import os |
||||
import platform |
||||
import sys |
||||
import uuid |
||||
|
||||
_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) |
||||
os.chdir(_ROOT) |
||||
|
||||
# How long to sleep before querying Resultstore API and uploading to bigquery |
||||
# (to let ResultStore finish writing results from the bazel invocation that has |
||||
# just finished). |
||||
_UPLOAD_RBE_RESULTS_DELAY_SECONDS = 60 |
||||
|
||||
|
||||
def _platform_string(): |
||||
"""Detect current platform""" |
||||
if platform.system() == 'Windows': |
||||
return 'windows' |
||||
elif platform.system()[:7] == 'MSYS_NT': |
||||
return 'windows' |
||||
elif platform.system() == 'Darwin': |
||||
return 'mac' |
||||
elif platform.system() == 'Linux': |
||||
return 'linux' |
||||
else: |
||||
return 'posix' |
||||
|
||||
|
||||
def _append_to_kokoro_bazel_invocations(invocation_id: str) -> None: |
||||
"""Kokoro can display "Bazel" result link on kokoro jobs if told so.""" |
||||
# to get "bazel" link for kokoro build, we need to upload |
||||
# the "bazel_invocation_ids" file with bazel invocation ID as artifact. |
||||
kokoro_artifacts_dir = os.getenv('KOKORO_ARTIFACTS_DIR') |
||||
if kokoro_artifacts_dir: |
||||
# append the bazel invocation UUID to the bazel_invocation_ids file. |
||||
with open(os.path.join(kokoro_artifacts_dir, 'bazel_invocation_ids'), |
||||
'a') as f: |
||||
f.write(invocation_id + '\n') |
||||
print( |
||||
'Added invocation ID %s to kokoro "bazel_invocation_ids" artifact' % |
||||
invocation_id, |
||||
file=sys.stderr) |
||||
else: |
||||
print( |
||||
'Skipped adding invocation ID %s to kokoro "bazel_invocation_ids" artifact' |
||||
% invocation_id, |
||||
file=sys.stderr) |
||||
pass |
||||
|
||||
|
||||
def _generate_junit_report_string(report_suite_name: str, invocation_id: str, |
||||
success: bool) -> None: |
||||
"""Generate sponge_log.xml formatted report, that will make the bazel invocation reachable as a target in resultstore UI / sponge.""" |
||||
bazel_invocation_url = 'https://source.cloud.google.com/results/invocations/%s' % invocation_id |
||||
package_name = report_suite_name |
||||
# set testcase name to invocation URL. That way, the link will be displayed in some form |
||||
# resultstore UI and sponge even in case the bazel invocation succeeds. |
||||
testcase_name = bazel_invocation_url |
||||
if success: |
||||
# unfortunately, neither resultstore UI nor sponge display the "system-err" output (or any other tags) |
||||
# on a passing test case. But at least we tried. |
||||
test_output_tag = '<system-err>PASSED. See invocation results here: %s</system-err>' % bazel_invocation_url |
||||
else: |
||||
# The failure output will be displayes in both resultstore UI and sponge when clicking on the failing testcase. |
||||
test_output_tag = '<failure message="Failure">FAILED. See bazel invocation results here: %s</failure>' % bazel_invocation_url |
||||
|
||||
lines = [ |
||||
'<testsuites>', |
||||
'<testsuite id="1" name="%s" package="%s">' % |
||||
(report_suite_name, package_name), |
||||
'<testcase name="%s">' % testcase_name, |
||||
test_output_tag, |
||||
'</testcase>' |
||||
'</testsuite>', |
||||
'</testsuites>', |
||||
] |
||||
return '\n'.join(lines) |
||||
|
||||
|
||||
def _create_bazel_wrapper(report_path: str, report_suite_name: str, |
||||
invocation_id: str, upload_results: bool) -> None: |
||||
"""Create a "bazel wrapper" script that will execute bazel with extra settings and postprocessing.""" |
||||
|
||||
os.makedirs(report_path, exist_ok=True) |
||||
|
||||
bazel_wrapper_filename = os.path.join(report_path, 'bazel_wrapper') |
||||
bazel_wrapper_bat_filename = bazel_wrapper_filename + '.bat' |
||||
bazel_rc_filename = os.path.join(report_path, 'bazel_wrapper.bazelrc') |
||||
|
||||
# put xml reports in a separate directory if requested by GRPC_TEST_REPORT_BASE_DIR |
||||
report_base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None) |
||||
xml_report_path = os.path.abspath( |
||||
os.path.join(report_base_dir, report_path |
||||
) if report_base_dir else report_path) |
||||
os.makedirs(xml_report_path, exist_ok=True) |
||||
|
||||
failing_report_filename = os.path.join(xml_report_path, 'sponge_log.xml') |
||||
success_report_filename = os.path.join(xml_report_path, |
||||
'success_log_to_rename.xml') |
||||
|
||||
if _platform_string() == 'windows': |
||||
workspace_status_command = 'tools/remote_build/workspace_status_kokoro.bat' |
||||
else: |
||||
workspace_status_command = 'tools/remote_build/workspace_status_kokoro.sh' |
||||
|
||||
# generate RC file with the bazel flags we want to use apply. |
||||
# Using an RC file solves problems with flag ordering in the wrapper. |
||||
# (e.g. some flags need to come after the build/test command) |
||||
with open(bazel_rc_filename, 'w') as f: |
||||
f.write('build --invocation_id="%s"\n' % invocation_id) |
||||
f.write('build --workspace_status_command="%s"\n' % |
||||
workspace_status_command) |
||||
|
||||
# generate "failing" and "success" report |
||||
# the "failing" is named as "sponge_log.xml", which is the name picked up by sponge/resultstore |
||||
# so the failing report will be used by default (unless we later replace the report with |
||||
# one that says "success"). That way if something goes wrong before bazel is run, |
||||
# there will at least be a "failing" target that indicates that (we really don't want silent failures). |
||||
with open(failing_report_filename, 'w') as f: |
||||
f.write( |
||||
_generate_junit_report_string(report_suite_name, |
||||
invocation_id, |
||||
success=False)) |
||||
with open(success_report_filename, 'w') as f: |
||||
f.write( |
||||
_generate_junit_report_string(report_suite_name, |
||||
invocation_id, |
||||
success=True)) |
||||
|
||||
# generate the bazel wrapper for linux/macos |
||||
with open(bazel_wrapper_filename, 'w') as f: |
||||
intro_lines = [ |
||||
'#!/bin/bash', |
||||
'set -ex', |
||||
'', |
||||
'tools/bazel --bazelrc="%s" "$@" || FAILED=true' % |
||||
bazel_rc_filename, |
||||
'', |
||||
] |
||||
|
||||
if upload_results: |
||||
upload_results_lines = [ |
||||
'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, |
||||
'PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s"' |
||||
% invocation_id, |
||||
'', |
||||
] |
||||
else: |
||||
upload_results_lines = [] |
||||
|
||||
outro_lines = [ |
||||
'if [ "$FAILED" != "" ]', |
||||
'then', |
||||
' exit 1', |
||||
'else', |
||||
' # success: plant the pre-generated xml report that says "success"', |
||||
' mv -f %s %s' % |
||||
(success_report_filename, failing_report_filename), |
||||
'fi', |
||||
] |
||||
|
||||
lines = [ |
||||
line + '\n' |
||||
for line in intro_lines + upload_results_lines + outro_lines |
||||
] |
||||
f.writelines(lines) |
||||
os.chmod(bazel_wrapper_filename, 0o775) # make the unix wrapper executable |
||||
|
||||
# generate bazel wrapper for windows |
||||
with open(bazel_wrapper_bat_filename, 'w') as f: |
||||
intro_lines = [ |
||||
'@echo on', |
||||
'', |
||||
'bazel --bazelrc="%s" %%*' % bazel_rc_filename, |
||||
'set BAZEL_EXITCODE=%errorlevel%', |
||||
'', |
||||
] |
||||
|
||||
if upload_results: |
||||
upload_results_lines = [ |
||||
'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS, |
||||
'python3 tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s" || exit /b 1' |
||||
% invocation_id, |
||||
'', |
||||
] |
||||
else: |
||||
upload_results_lines = [] |
||||
|
||||
outro_lines = [ |
||||
'if %BAZEL_EXITCODE% == 0 (', |
||||
' @rem success: plant the pre-generated xml report that says "success"', |
||||
' mv -f %s %s' % |
||||
(success_report_filename, failing_report_filename), |
||||
')', |
||||
'exit /b %BAZEL_EXITCODE%', |
||||
] |
||||
|
||||
lines = [ |
||||
line + '\n' |
||||
for line in intro_lines + upload_results_lines + outro_lines |
||||
] |
||||
f.writelines(lines) |
||||
|
||||
print('Bazel invocation ID: %s' % invocation_id, file=sys.stderr) |
||||
print('Upload test results to BigQuery after bazel runs: %s' % |
||||
upload_results, |
||||
file=sys.stderr) |
||||
print('Generated bazel wrapper: %s' % bazel_wrapper_filename, |
||||
file=sys.stderr) |
||||
print('Generated bazel wrapper: %s' % bazel_wrapper_bat_filename, |
||||
file=sys.stderr) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
# parse command line |
||||
argp = argparse.ArgumentParser( |
||||
description= |
||||
'Generate bazel wrapper to help with bazel test reports in CI.') |
||||
argp.add_argument( |
||||
'--report_path', |
||||
required=True, |
||||
type=str, |
||||
help= |
||||
'Path under which the bazel wrapper and other files are going to be generated' |
||||
) |
||||
argp.add_argument('--report_suite_name', |
||||
default='bazel_invocations', |
||||
type=str, |
||||
help='Test suite name to use in generated XML report') |
||||
args = argp.parse_args() |
||||
|
||||
# generate new bazel invocation ID |
||||
invocation_id = str(uuid.uuid4()) |
||||
|
||||
report_path = args.report_path |
||||
report_suite_name = args.report_suite_name |
||||
upload_results = True if os.getenv('UPLOAD_TEST_RESULTS') else False |
||||
|
||||
_append_to_kokoro_bazel_invocations(invocation_id) |
||||
_create_bazel_wrapper(report_path, report_suite_name, invocation_id, |
||||
upload_results) |
||||
Loading…
Reference in new issue