mirror of https://github.com/grpc/grpc.git
[test] Semi-automatic flaky test tracking (#30638)
* [test] Semi-automatic flaky test tracking * fix * Automated change: Fix sanity tests * Update update_flakes.py * fix formatting * Automated change: Fix sanity tests Co-authored-by: ctiller <ctiller@users.noreply.github.com>pull/30667/head
Craig Tiller
2 years ago
committed by
GitHub
parent
fedba4654d
commit
f573fd44f9
30 changed files with 543 additions and 507 deletions
@ -0,0 +1,168 @@ |
|||||||
|
# Copyright 2022 gRPC authors. |
||||||
|
# |
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
# you may not use this file except in compliance with the License. |
||||||
|
# You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
"""A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs.""" |
||||||
|
FLAKY_TESTS = [ |
||||||
|
"h2_census_test@bad_ping", |
||||||
|
"h2_census_test@grpc_authz", |
||||||
|
"h2_census_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_census_test@retry_too_many_attempts", |
||||||
|
"h2_census_test@retry_transparent_mcs", |
||||||
|
"h2_census_test@shutdown_finishes_calls", |
||||||
|
"h2_compress_test@grpc_authz", |
||||||
|
"h2_compress_test@keepalive_timeout", |
||||||
|
"h2_compress_test@max_connection_age", |
||||||
|
"h2_compress_test@max_connection_idle", |
||||||
|
"h2_compress_test@retry_cancellation", |
||||||
|
"h2_fakesec_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_fakesec_test@retry_server_pushback_disabled", |
||||||
|
"h2_fakesec_test@retry_too_many_attempts", |
||||||
|
"h2_fakesec_test@shutdown_finishes_calls", |
||||||
|
"h2_fd_test@max_connection_age", |
||||||
|
"h2_full+pipe_test@max_connection_idle", |
||||||
|
"h2_full_no_retry_test@bad_ping", |
||||||
|
"h2_full_no_retry_test@grpc_authz", |
||||||
|
"h2_full_no_retry_test@hpack_size", |
||||||
|
"h2_full_no_retry_test@max_connection_age", |
||||||
|
"h2_full_no_retry_test@max_connection_idle", |
||||||
|
"h2_full_test@bad_ping", |
||||||
|
"h2_full_test@keepalive_timeout", |
||||||
|
"h2_full_test@max_connection_age", |
||||||
|
"h2_full_test@max_connection_idle", |
||||||
|
"h2_full_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_full_test@retry_recv_trailing_metadata_error", |
||||||
|
"h2_full_test@retry_send_initial_metadata_refs", |
||||||
|
"h2_full_test@retry_server_pushback_delay", |
||||||
|
"h2_http_proxy_test@compressed_payload", |
||||||
|
"h2_http_proxy_test@connectivity", |
||||||
|
"h2_http_proxy_test@grpc_authz", |
||||||
|
"h2_http_proxy_test@hpack_size", |
||||||
|
"h2_http_proxy_test@max_connection_age", |
||||||
|
"h2_http_proxy_test@max_connection_idle", |
||||||
|
"h2_http_proxy_test@ping_pong_streaming", |
||||||
|
"h2_http_proxy_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_insecure_test@bad_ping", |
||||||
|
"h2_insecure_test@grpc_authz", |
||||||
|
"h2_insecure_test@max_connection_age", |
||||||
|
"h2_insecure_test@max_connection_idle", |
||||||
|
"h2_insecure_test@retry", |
||||||
|
"h2_insecure_test@retry_send_initial_metadata_refs", |
||||||
|
"h2_insecure_test@retry_server_pushback_disabled", |
||||||
|
"h2_local_abstract_uds_percent_encoded_test@max_connection_idle", |
||||||
|
"h2_local_ipv4_test@bad_ping", |
||||||
|
"h2_local_ipv4_test@max_connection_idle", |
||||||
|
"h2_local_ipv4_test@retry", |
||||||
|
"h2_local_ipv4_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_local_ipv4_test@retry_send_initial_metadata_refs", |
||||||
|
"h2_local_ipv4_test@retry_streaming2", |
||||||
|
"h2_local_ipv6_test@bad_ping", |
||||||
|
"h2_local_ipv6_test@cancel_after_accept", |
||||||
|
"h2_local_ipv6_test@max_connection_idle", |
||||||
|
"h2_local_ipv6_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_local_ipv6_test@retry_server_pushback_disabled", |
||||||
|
"h2_local_ipv6_test@shutdown_finishes_calls", |
||||||
|
"h2_local_uds_percent_encoded_test@bad_ping", |
||||||
|
"h2_local_uds_percent_encoded_test@connectivity", |
||||||
|
"h2_local_uds_percent_encoded_test@max_connection_idle", |
||||||
|
"h2_local_uds_percent_encoded_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_local_uds_percent_encoded_test@retry_streaming2", |
||||||
|
"h2_local_uds_percent_encoded_test@retry_transparent_mcs", |
||||||
|
"h2_local_uds_test@bad_ping", |
||||||
|
"h2_local_uds_test@grpc_authz", |
||||||
|
"h2_local_uds_test@max_connection_age", |
||||||
|
"h2_local_uds_test@max_connection_idle", |
||||||
|
"h2_local_uds_test@retry_server_pushback_disabled", |
||||||
|
"h2_oauth2_test@bad_ping", |
||||||
|
"h2_oauth2_test@connectivity", |
||||||
|
"h2_oauth2_test@filter_context", |
||||||
|
"h2_oauth2_test@grpc_authz", |
||||||
|
"h2_oauth2_test@max_connection_age", |
||||||
|
"h2_oauth2_test@max_connection_idle", |
||||||
|
"h2_oauth2_test@retry_server_pushback_delay", |
||||||
|
"h2_oauth2_test@retry_server_pushback_disabled", |
||||||
|
"h2_oauth2_test@retry_too_many_attempts", |
||||||
|
"h2_proxy_test@grpc_authz", |
||||||
|
"h2_proxy_test@retry_send_initial_metadata_refs", |
||||||
|
"h2_proxy_test@shutdown_finishes_calls", |
||||||
|
"h2_sockpair_1byte_test@max_connection_age", |
||||||
|
"h2_sockpair_1byte_test@shutdown_finishes_calls", |
||||||
|
"h2_sockpair_test@grpc_authz", |
||||||
|
"h2_sockpair_test@max_connection_age", |
||||||
|
"h2_ssl_cred_reload_test@bad_ping", |
||||||
|
"h2_ssl_cred_reload_test@client_streaming", |
||||||
|
"h2_ssl_cred_reload_test@connectivity", |
||||||
|
"h2_ssl_cred_reload_test@max_connection_idle", |
||||||
|
"h2_ssl_cred_reload_test@retry", |
||||||
|
"h2_ssl_cred_reload_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_ssl_cred_reload_test@retry_server_pushback_delay", |
||||||
|
"h2_ssl_cred_reload_test@retry_too_many_attempts", |
||||||
|
"h2_ssl_cred_reload_test@simple_delayed_request", |
||||||
|
"h2_ssl_proxy_test@disappearing_server", |
||||||
|
"h2_ssl_proxy_test@retry_cancellation", |
||||||
|
"h2_ssl_proxy_test@retry_server_pushback_delay", |
||||||
|
"h2_ssl_proxy_test@retry_too_many_attempts", |
||||||
|
"h2_ssl_proxy_test@shutdown_finishes_calls", |
||||||
|
"h2_ssl_test@bad_ping", |
||||||
|
"h2_ssl_test@compressed_payload", |
||||||
|
"h2_ssl_test@grpc_authz", |
||||||
|
"h2_ssl_test@max_connection_idle", |
||||||
|
"h2_ssl_test@retry", |
||||||
|
"h2_ssl_test@retry_cancellation", |
||||||
|
"h2_ssl_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_ssl_test@retry_send_initial_metadata_refs", |
||||||
|
"h2_ssl_test@retry_server_pushback_disabled", |
||||||
|
"h2_ssl_test@retry_too_many_attempts", |
||||||
|
"h2_ssl_test@shutdown_finishes_calls", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@cancel_after_invoke", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@connectivity", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@max_connection_idle", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@retry_cancellation", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@retry_streaming", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@retry_streaming2", |
||||||
|
"h2_tls_certwatch_async_tls1_3_test@simple_delayed_request", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@connectivity", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@max_connection_idle", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@retry_cancellation", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@retry_server_pushback_delay", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@simple_delayed_request", |
||||||
|
"h2_tls_certwatch_sync_tls1_2_test@write_buffering", |
||||||
|
"h2_tls_simple_test@bad_ping", |
||||||
|
"h2_tls_simple_test@connectivity", |
||||||
|
"h2_tls_simple_test@max_connection_idle", |
||||||
|
"h2_tls_simple_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_tls_simple_test@simple_delayed_request", |
||||||
|
"h2_tls_static_async_tls1_3_test@connectivity", |
||||||
|
"h2_tls_static_async_tls1_3_test@max_connection_idle", |
||||||
|
"h2_tls_static_async_tls1_3_test@retry_cancellation", |
||||||
|
"h2_tls_static_async_tls1_3_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_tls_static_async_tls1_3_test@simple_delayed_request", |
||||||
|
"h2_tls_test@bad_ping", |
||||||
|
"h2_tls_test@connectivity", |
||||||
|
"h2_tls_test@grpc_authz", |
||||||
|
"h2_tls_test@hpack_size", |
||||||
|
"h2_tls_test@invoke_large_request", |
||||||
|
"h2_tls_test@keepalive_timeout", |
||||||
|
"h2_tls_test@max_connection_idle", |
||||||
|
"h2_tls_test@resource_quota_server", |
||||||
|
"h2_tls_test@retry", |
||||||
|
"h2_tls_test@retry_cancellation", |
||||||
|
"h2_tls_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_tls_test@retry_streaming", |
||||||
|
"h2_tls_test@retry_streaming2", |
||||||
|
"h2_tls_test@simple_delayed_request", |
||||||
|
"h2_uds_test@bad_ping", |
||||||
|
"h2_uds_test@max_connection_idle", |
||||||
|
"h2_uds_test@retry_per_attempt_recv_timeout", |
||||||
|
"h2_uds_test@retry_streaming", |
||||||
|
"h2_uds_test@retry_transparent_mcs", |
||||||
|
] |
||||||
@ -0,0 +1,30 @@ |
|||||||
|
# Copyright 2022 gRPC authors. |
||||||
|
# |
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
# you may not use this file except in compliance with the License. |
||||||
|
# You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
|
||||||
|
import subprocess |
||||||
|
import sys |
||||||
|
import tempfile |
||||||
|
|
||||||
|
|
||||||
|
def run_buildozer(buildozer_commands): |
||||||
|
if not buildozer_commands: |
||||||
|
return |
||||||
|
ok_statuses = (0, 3) |
||||||
|
temp = tempfile.NamedTemporaryFile() |
||||||
|
open(temp.name, 'w').write('\n'.join(buildozer_commands)) |
||||||
|
c = ['tools/distrib/buildozer.sh', '-f', temp.name] |
||||||
|
r = subprocess.call(c) |
||||||
|
if r not in ok_statuses: |
||||||
|
print('{} failed with status {}'.format(c, r)) |
||||||
|
sys.exit(1) |
||||||
@ -0,0 +1,102 @@ |
|||||||
|
#!/usr/bin/env python3 |
||||||
|
|
||||||
|
# Copyright 2022 gRPC authors. |
||||||
|
# |
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
# you may not use this file except in compliance with the License. |
||||||
|
# You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
|
||||||
|
import os |
||||||
|
import re |
||||||
|
import subprocess |
||||||
|
import sys |
||||||
|
|
||||||
|
from google.cloud import bigquery |
||||||
|
import run_buildozer |
||||||
|
import update_flakes_query |
||||||
|
|
||||||
|
lookback_hours = 24 * 7 * 4 |
||||||
|
|
||||||
|
|
||||||
|
def include_test(test): |
||||||
|
if '@' in test: |
||||||
|
return False |
||||||
|
if test.startswith("//test/cpp/qps:"): |
||||||
|
return False |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
TEST_DIRS = ['test/core', 'test/cpp'] |
||||||
|
tests = {} |
||||||
|
already_flaky = set() |
||||||
|
for test_dir in TEST_DIRS: |
||||||
|
for line in subprocess.check_output( |
||||||
|
['bazel', 'query', 'tests({}/...)'.format(test_dir)]).splitlines(): |
||||||
|
test = line.strip().decode('utf-8') |
||||||
|
if not include_test(test): |
||||||
|
continue |
||||||
|
tests[test] = False |
||||||
|
for test_dir in TEST_DIRS: |
||||||
|
for line in subprocess.check_output( |
||||||
|
['bazel', 'query', |
||||||
|
'attr(flaky, 1, tests({}/...))'.format(test_dir)]).splitlines(): |
||||||
|
test = line.strip().decode('utf-8') |
||||||
|
if not include_test(test): |
||||||
|
continue |
||||||
|
already_flaky.add(test) |
||||||
|
|
||||||
|
flaky_e2e = set() |
||||||
|
|
||||||
|
client = bigquery.Client() |
||||||
|
for row in client.query( |
||||||
|
update_flakes_query.QUERY.format( |
||||||
|
lookback_hours=lookback_hours)).result(): |
||||||
|
if row.test_binary not in tests: |
||||||
|
m = re.match(r'^//test/core/end2end:([^@]*)@([^@]*)(.*)', |
||||||
|
row.test_binary) |
||||||
|
if m: |
||||||
|
flaky_e2e.add('{}@{}{}'.format(m.group(1), m.group(2), m.group(3))) |
||||||
|
print("will mark end2end test {} as flaky".format(row.test_binary)) |
||||||
|
else: |
||||||
|
print("skip obsolete test {}".format(row.test_binary)) |
||||||
|
continue |
||||||
|
print("will mark {} as flaky".format(row.test_binary)) |
||||||
|
tests[row.test_binary] = True |
||||||
|
|
||||||
|
buildozer_commands = [] |
||||||
|
for test, flaky in sorted(tests.items()): |
||||||
|
if flaky: |
||||||
|
buildozer_commands.append('set flaky True|{}'.format(test)) |
||||||
|
elif test in already_flaky: |
||||||
|
buildozer_commands.append('remove flaky|{}'.format(test)) |
||||||
|
|
||||||
|
with open('test/core/end2end/flaky.bzl', 'w') as f: |
||||||
|
with open(sys.argv[0]) as my_source: |
||||||
|
for line in my_source: |
||||||
|
if line[0] != '#': |
||||||
|
break |
||||||
|
for line in my_source: |
||||||
|
if line[0] == '#': |
||||||
|
print(line.strip(), file=f) |
||||||
|
break |
||||||
|
for line in my_source: |
||||||
|
if line[0] != '#': |
||||||
|
break |
||||||
|
print(line.strip(), file=f) |
||||||
|
print( |
||||||
|
"\"\"\"A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs.\"\"\"", |
||||||
|
file=f) |
||||||
|
print("FLAKY_TESTS = [", file=f) |
||||||
|
for line in sorted(list(flaky_e2e)): |
||||||
|
print(" \"{}\",".format(line), file=f) |
||||||
|
print("]", file=f) |
||||||
|
|
||||||
|
run_buildozer.run_buildozer(buildozer_commands) |
||||||
@ -0,0 +1,147 @@ |
|||||||
|
# Copyright 2022 gRPC authors. |
||||||
|
# |
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||||
|
# you may not use this file except in compliance with the License. |
||||||
|
# You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, software |
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, |
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||||
|
# See the License for the specific language governing permissions and |
||||||
|
# limitations under the License. |
||||||
|
|
||||||
|
QUERY = """ |
||||||
|
#standardSQL |
||||||
|
|
||||||
|
-- See https://console.cloud.google.com/bigquery?sq=830293263384:5a8549832dfb48d9b2c04312a4ae3181 for the original query |
||||||
|
|
||||||
|
WITH |
||||||
|
|
||||||
|
runs AS ( |
||||||
|
SELECT |
||||||
|
RTRIM(LTRIM(REGEXP_REPLACE(test_target, r'(@poller=.+)', ''))) AS test_binary, |
||||||
|
REGEXP_EXTRACT(test_target, r'poller=(\w+)') AS poll_strategy, |
||||||
|
job_name, |
||||||
|
test_target, |
||||||
|
test_class_name, |
||||||
|
CASE |
||||||
|
# in case of timeout / retry / segfault the "test_case" fields will contain weird stuff |
||||||
|
# e.g. "test_shard2_run0_attempt0" or "shard_2/20" |
||||||
|
# when aggregating, we want to display all of these as a single category of problems |
||||||
|
WHEN test_case like 'test_shard_%_run%_attempt%' THEN 'CANNOT_DETERMINE' |
||||||
|
WHEN test_case like '%shard_%/%' THEN 'CANNOT_DETERMINE' |
||||||
|
# when test_case looks suspiciously like test_target |
||||||
|
# its value is probably meaningless and it means that the entire target has failed |
||||||
|
# e.g. test_target="//test/cpp/client:destroy_grpclb_channel_with_active_connect_stress_test" and test_case="test/cpp/client/destroy_grpclb_channel_with_active_connect_stress_test.exe" |
||||||
|
WHEN STRPOS(test_case, REPLACE(SUBSTR(test_target, 3), ":", "/")) != 0 THEN 'CANNOT_DETERMINE' |
||||||
|
ELSE test_case |
||||||
|
END AS test_case, |
||||||
|
result, |
||||||
|
build_id, |
||||||
|
timestamp |
||||||
|
FROM |
||||||
|
`grpc-testing.jenkins_test_results.rbe_test_results` |
||||||
|
WHERE |
||||||
|
DATETIME_DIFF(CURRENT_DATETIME(), |
||||||
|
dateTIME(timestamp), |
||||||
|
HOUR) < {lookback_hours} |
||||||
|
), |
||||||
|
|
||||||
|
results_counts_per_build AS ( |
||||||
|
SELECT |
||||||
|
test_binary, |
||||||
|
#test_target, # aggregate data over all pollers |
||||||
|
test_class_name, |
||||||
|
test_case, |
||||||
|
SUM(SAFE_CAST(result != 'PASSED' |
||||||
|
AND result != 'SKIPPED' AS INT64)) AS runs_failed, |
||||||
|
SUM(SAFE_CAST(result != 'SKIPPED' AS INT64)) AS runs_total, |
||||||
|
job_name, |
||||||
|
build_id |
||||||
|
FROM |
||||||
|
runs |
||||||
|
GROUP BY |
||||||
|
test_binary, |
||||||
|
test_class_name, |
||||||
|
test_case, |
||||||
|
job_name, |
||||||
|
build_id), |
||||||
|
|
||||||
|
builds_with_missing_cannot_determine_testcase_entry AS ( |
||||||
|
SELECT |
||||||
|
test_binary, |
||||||
|
job_name, |
||||||
|
build_id, |
||||||
|
FROM |
||||||
|
results_counts_per_build |
||||||
|
GROUP BY |
||||||
|
test_binary, |
||||||
|
job_name, |
||||||
|
build_id |
||||||
|
HAVING COUNTIF(test_case = 'CANNOT_DETERMINE') = 0 |
||||||
|
), |
||||||
|
|
||||||
|
# for each test target and build, generate a fake entry with "CANNOT_DETERMINE" test_case |
||||||
|
# if not already present. |
||||||
|
# this is because in many builds, there will be no "CANNOT_DETERMINE" entry |
||||||
|
# and we want to avoid skewing the statistics |
||||||
|
results_counts_per_build_with_fake_cannot_determine_test_case_entries AS ( |
||||||
|
(SELECT * FROM results_counts_per_build) |
||||||
|
UNION ALL |
||||||
|
(SELECT |
||||||
|
test_binary, |
||||||
|
'' AS test_class_name, # when test_case is 'CANNOT_DETERMINE', test class is empty string |
||||||
|
'CANNOT_DETERMINE' AS test_case, # see table "runs" |
||||||
|
0 AS runs_failed, |
||||||
|
1 AS runs_total, |
||||||
|
job_name, |
||||||
|
build_id |
||||||
|
FROM |
||||||
|
builds_with_missing_cannot_determine_testcase_entry) |
||||||
|
), |
||||||
|
|
||||||
|
results_counts AS ( |
||||||
|
SELECT |
||||||
|
test_binary, |
||||||
|
test_class_name, |
||||||
|
test_case, |
||||||
|
job_name, |
||||||
|
SUM(runs_failed) AS runs_failed, |
||||||
|
SUM(runs_total) AS runs_total, |
||||||
|
SUM(SAFE_CAST(runs_failed > 0 AS INT64)) AS builds_failed, |
||||||
|
COUNT(build_id) AS builds_total, |
||||||
|
STRING_AGG(CASE |
||||||
|
WHEN runs_failed > 0 THEN 'X' |
||||||
|
ELSE '_' END, '' |
||||||
|
ORDER BY |
||||||
|
build_id ASC) AS build_failure_pattern, |
||||||
|
FORMAT("%T", ARRAY_AGG(build_id |
||||||
|
ORDER BY |
||||||
|
build_id ASC)) AS builds |
||||||
|
FROM |
||||||
|
#results_counts_per_build |
||||||
|
results_counts_per_build_with_fake_cannot_determine_test_case_entries |
||||||
|
GROUP BY |
||||||
|
test_binary, |
||||||
|
test_class_name, |
||||||
|
test_case, |
||||||
|
job_name |
||||||
|
HAVING |
||||||
|
runs_failed > 0) |
||||||
|
|
||||||
|
SELECT |
||||||
|
ROUND(100*builds_failed / builds_total, 2) AS pct_builds_failed, |
||||||
|
ROUND(100*runs_failed / runs_total, 2) AS pct_runs_failed, |
||||||
|
test_binary, |
||||||
|
test_class_name, |
||||||
|
test_case, |
||||||
|
job_name, |
||||||
|
build_failure_pattern |
||||||
|
|
||||||
|
FROM |
||||||
|
results_counts |
||||||
|
ORDER BY |
||||||
|
pct_builds_failed DESC |
||||||
|
""" |
||||||
Loading…
Reference in new issue