Update flaky tests (#30873)

* Update flaky tests list

* Automated change: Fix sanity tests

Co-authored-by: ctiller <ctiller@users.noreply.github.com>
pull/30839/head
Craig Tiller 2 years ago committed by GitHub
parent 2bb6097412
commit b463cabeec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 12
      build_autogenerated.yaml
  2. 1
      test/core/end2end/BUILD
  3. 135
      test/core/end2end/flaky.bzl
  4. 1
      test/core/event_engine/posix/BUILD
  5. 1
      test/core/fling/BUILD
  6. 1
      test/core/handshake/BUILD
  7. 1
      test/core/http/BUILD
  8. 1
      test/core/iomgr/BUILD
  9. 2
      test/core/security/BUILD
  10. 4
      test/cpp/end2end/BUILD
  11. 2
      test/cpp/end2end/xds/BUILD
  12. 1
      test/cpp/server/load_reporter/BUILD
  13. 2
      tools/distrib/update_flakes.py
  14. 274
      tools/run_tests/generated/tests.json

@ -3458,7 +3458,6 @@ targets:
- mac
- name: fling_stream_test
build: test
run: false
language: c
headers:
- test/core/end2end/data/ssl_test_data.h
@ -3543,7 +3542,6 @@ targets:
- mac
- name: goaway_server_test
build: test
run: false
language: c
headers:
- test/core/end2end/cq_verifier.h
@ -5020,7 +5018,6 @@ targets:
- name: client_interceptors_end2end_test
gtest: true
build: test
run: false
language: c++
headers:
- test/cpp/end2end/interceptors_util.h
@ -5066,7 +5063,6 @@ targets:
- name: client_ssl_test
gtest: true
build: test
run: false
language: c++
headers: []
src:
@ -5748,7 +5744,6 @@ targets:
- name: event_poller_posix_test
gtest: true
build: test
run: false
language: c++
headers:
- src/core/lib/event_engine/common_closures.h
@ -6706,7 +6701,6 @@ targets:
- name: grpc_tls_certificate_provider_test
gtest: true
build: test
run: false
language: c++
headers:
- test/core/util/cmdline.h
@ -6809,7 +6803,6 @@ targets:
- name: grpc_tls_credentials_options_test
gtest: true
build: test
run: false
language: c++
headers:
- test/core/util/cmdline.h
@ -7170,7 +7163,6 @@ targets:
- name: httpscli_test
gtest: true
build: test
run: false
language: c++
headers:
- test/core/http/httpcli_test_util.h
@ -7932,7 +7924,6 @@ targets:
- name: orca_service_end2end_test
gtest: true
build: test
run: false
language: c++
headers: []
src:
@ -9462,7 +9453,6 @@ targets:
- name: stranded_event_test
gtest: true
build: test
run: false
language: c++
headers:
- test/core/end2end/cq_verifier.h
@ -10957,7 +10947,6 @@ targets:
- name: xds_csds_end2end_test
gtest: true
build: test
run: false
language: c++
headers:
- src/cpp/server/csds/csds.h
@ -11343,7 +11332,6 @@ targets:
- name: xds_rls_end2end_test
gtest: true
build: test
run: false
language: c++
headers:
- test/cpp/end2end/counted_service.h

@ -168,7 +168,6 @@ grpc_cc_test(
"absl/status:statusor",
"absl/strings",
],
flaky = True,
language = "C++",
deps = [
"cq_verifier",

@ -13,181 +13,50 @@
# limitations under the License.
"""A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs."""
FLAKY_TESTS = [
"h2_census_test@grpc_authz",
"h2_census_test@retry_server_pushback_delay",
"h2_census_test@retry_streaming2",
"h2_census_test@retry_too_many_attempts",
"h2_census_test@retry_transparent_mcs",
"h2_census_test@shutdown_finishes_calls",
"h2_compress_test@bad_ping",
"h2_compress_test@grpc_authz",
"h2_compress_test@max_connection_age",
"h2_compress_test@max_connection_idle",
"h2_compress_test@retry_cancellation",
"h2_compress_test@retry_per_attempt_recv_timeout",
"h2_fakesec_test@bad_ping",
"h2_fakesec_test@max_connection_idle",
"h2_fakesec_test@retry_per_attempt_recv_timeout",
"h2_fakesec_test@retry_server_pushback_disabled",
"h2_fakesec_test@retry_too_many_attempts",
"h2_fakesec_test@shutdown_finishes_calls",
"h2_fd_test@filtered_metadata",
"h2_fd_test@max_connection_age",
"h2_full+pipe_test@max_connection_age",
"h2_full+pipe_test@max_connection_idle",
"h2_full_no_retry_test@bad_ping",
"h2_full_no_retry_test@grpc_authz",
"h2_full_no_retry_test@hpack_size",
"h2_full_no_retry_test@max_connection_age",
"h2_full_no_retry_test@max_connection_idle",
"h2_full_test@bad_ping",
"h2_full_test@keepalive_timeout",
"h2_full_test@max_connection_age",
"h2_full_test@max_connection_idle",
"h2_full_test@retry_per_attempt_recv_timeout",
"h2_full_test@retry_recv_trailing_metadata_error",
"h2_full_test@retry_server_pushback_delay",
"h2_full_test@retry_transparent_mcs",
"h2_http_proxy_test@compressed_payload",
"h2_http_proxy_test@connectivity",
"h2_http_proxy_test@grpc_authz",
"h2_http_proxy_test@hpack_size",
"h2_http_proxy_test@max_connection_age",
"h2_http_proxy_test@max_connection_idle",
"h2_http_proxy_test@ping_pong_streaming",
"h2_http_proxy_test@retry_per_attempt_recv_timeout",
"h2_http_proxy_test@retry_server_pushback_disabled",
"h2_insecure_test@bad_ping",
"h2_insecure_test@grpc_authz",
"h2_insecure_test@max_connection_age",
"h2_insecure_test@max_connection_idle",
"h2_insecure_test@retry",
"h2_insecure_test@retry_server_pushback_disabled",
"h2_insecure_test@retry_streaming",
"h2_local_abstract_uds_percent_encoded_test@max_connection_idle",
"h2_local_ipv4_test@bad_ping",
"h2_local_ipv4_test@max_connection_idle",
"h2_local_ipv4_test@retry",
"h2_local_ipv4_test@retry_per_attempt_recv_timeout",
"h2_local_ipv4_test@retry_server_pushback_delay",
"h2_local_ipv6_test@cancel_after_accept",
"h2_local_ipv6_test@grpc_authz",
"h2_local_ipv6_test@max_connection_idle",
"h2_local_ipv6_test@retry_per_attempt_recv_timeout",
"h2_local_ipv6_test@retry_send_initial_metadata_refs",
"h2_local_ipv6_test@retry_server_pushback_disabled",
"h2_local_ipv6_test@shutdown_finishes_calls",
"h2_local_uds_percent_encoded_test@bad_ping",
"h2_local_uds_percent_encoded_test@connectivity",
"h2_local_uds_percent_encoded_test@grpc_authz",
"h2_local_uds_percent_encoded_test@max_connection_idle",
"h2_local_uds_percent_encoded_test@ping",
"h2_local_uds_percent_encoded_test@retry_per_attempt_recv_timeout",
"h2_local_uds_percent_encoded_test@retry_server_pushback_delay",
"h2_local_uds_percent_encoded_test@retry_streaming2",
"h2_local_uds_percent_encoded_test@retry_transparent_mcs",
"h2_local_uds_percent_encoded_test@shutdown_finishes_calls",
"h2_local_uds_test@bad_ping",
"h2_local_uds_test@grpc_authz",
"h2_local_uds_test@max_connection_age",
"h2_local_uds_test@max_connection_idle",
"h2_local_uds_test@retry_per_attempt_recv_timeout",
"h2_local_uds_test@retry_server_pushback_delay",
"h2_local_uds_test@retry_server_pushback_disabled",
"h2_oauth2_test@bad_ping",
"h2_oauth2_test@connectivity",
"h2_oauth2_test@filter_context",
"h2_oauth2_test@grpc_authz",
"h2_oauth2_test@max_connection_age",
"h2_oauth2_test@max_connection_idle",
"h2_oauth2_test@retry_too_many_attempts",
"h2_oauth2_tls12_test@max_connection_idle",
"h2_oauth2_tls12_test@retry_per_attempt_recv_timeout",
"h2_oauth2_tls12_test@retry_server_pushback_delay",
"h2_oauth2_tls13_test@bad_ping",
"h2_proxy_test@grpc_authz",
"h2_proxy_test@retry_send_initial_metadata_refs",
"h2_proxy_test@retry_streaming2",
"h2_proxy_test@shutdown_finishes_calls",
"h2_proxy_test@cancel_with_status",
"h2_sockpair_1byte_test@max_connection_age",
"h2_sockpair_1byte_test@shutdown_finishes_calls",
"h2_sockpair_test@max_connection_age",
"h2_ssl_cred_reload_test@bad_ping",
"h2_ssl_cred_reload_test@client_streaming",
"h2_ssl_cred_reload_test@connectivity",
"h2_ssl_cred_reload_test@max_connection_idle",
"h2_ssl_cred_reload_test@retry",
"h2_ssl_cred_reload_test@retry_per_attempt_recv_timeout",
"h2_ssl_cred_reload_test@retry_server_pushback_delay",
"h2_ssl_cred_reload_test@simple_delayed_request",
"h2_ssl_cred_reload_tls12_test@bad_ping",
"h2_ssl_cred_reload_tls12_test@max_connection_age",
"h2_ssl_cred_reload_tls12_test@max_connection_idle",
"h2_ssl_cred_reload_tls12_test@retry_server_pushback_delay",
"h2_ssl_cred_reload_tls12_test@retry_too_many_attempts",
"h2_ssl_cred_reload_tls12_test@shutdown_finishes_calls",
"h2_ssl_cred_reload_tls13_test@retry_per_attempt_recv_timeout",
"h2_ssl_proxy_test@retry_exceeds_buffer_size_in_delay",
"h2_ssl_proxy_test@retry_server_pushback_delay",
"h2_ssl_proxy_test@shutdown_finishes_calls",
"h2_ssl_test@bad_ping",
"h2_ssl_test@compressed_payload",
"h2_ssl_proxy_test@retry_transparent_not_sent_on_wire",
"h2_ssl_test@max_connection_idle",
"h2_ssl_test@retry_cancellation",
"h2_ssl_test@retry_per_attempt_recv_timeout",
"h2_ssl_test@retry_send_initial_metadata_refs",
"h2_ssl_test@retry_server_pushback_disabled",
"h2_ssl_test@retry_too_many_attempts",
"h2_ssl_test@shutdown_finishes_calls",
"h2_ssl_tls12_test@client_streaming",
"h2_ssl_tls12_test@max_connection_idle",
"h2_ssl_tls13_test@retry_per_attempt_recv_timeout",
"h2_tls_certwatch_async_tls1_3_test@cancel_after_invoke",
"h2_tls_certwatch_async_tls1_3_test@connectivity",
"h2_tls_certwatch_async_tls1_3_test@max_connection_idle",
"h2_tls_certwatch_async_tls1_3_test@retry_cancellation",
"h2_tls_certwatch_async_tls1_3_test@retry_streaming",
"h2_tls_certwatch_async_tls1_3_test@retry_streaming2",
"h2_tls_certwatch_async_tls1_3_test@simple_delayed_request",
"h2_tls_certwatch_sync_tls1_2_test@connectivity",
"h2_tls_certwatch_sync_tls1_2_test@max_connection_idle",
"h2_tls_certwatch_sync_tls1_2_test@retry_cancellation",
"h2_tls_certwatch_sync_tls1_2_test@retry_server_pushback_delay",
"h2_tls_certwatch_sync_tls1_2_test@simple_delayed_request",
"h2_tls_certwatch_sync_tls1_2_test@write_buffering",
"h2_tls_simple_test@bad_ping",
"h2_tls_simple_test@connectivity",
"h2_tls_simple_test@grpc_authz",
"h2_tls_simple_test@max_connection_idle",
"h2_tls_simple_test@retry_per_attempt_recv_timeout",
"h2_tls_simple_test@simple_delayed_request",
"h2_tls_simple_test@write_buffering",
"h2_tls_static_async_tls1_3_test@connectivity",
"h2_tls_static_async_tls1_3_test@max_connection_idle",
"h2_tls_static_async_tls1_3_test@ping",
"h2_tls_static_async_tls1_3_test@retry_cancellation",
"h2_tls_static_async_tls1_3_test@retry_per_attempt_recv_timeout",
"h2_tls_static_async_tls1_3_test@retry_server_pushback_delay",
"h2_tls_static_async_tls1_3_test@retry_streaming",
"h2_tls_static_async_tls1_3_test@simple_delayed_request",
"h2_tls_test@bad_ping",
"h2_tls_test@connectivity",
"h2_tls_test@grpc_authz",
"h2_tls_test@hpack_size",
"h2_tls_test@invoke_large_request",
"h2_tls_test@keepalive_timeout",
"h2_tls_test@max_connection_idle",
"h2_tls_test@resource_quota_server",
"h2_tls_test@retry",
"h2_tls_test@retry_cancellation",
"h2_tls_test@retry_per_attempt_recv_timeout",
"h2_tls_test@retry_streaming2",
"h2_tls_test@simple_delayed_request",
"h2_uds_abstract_test@max_connection_age",
"h2_uds_test@bad_ping",
"h2_uds_test@max_connection_idle",
"h2_uds_test@retry_per_attempt_recv_timeout",
"h2_uds_test@retry_server_pushback_delay",
"h2_uds_test@retry_streaming",
"h2_uds_test@retry_transparent_mcs",
]

@ -65,7 +65,6 @@ grpc_cc_test(
name = "event_poller_posix_test",
srcs = ["event_poller_posix_test.cc"],
external_deps = ["gtest"],
flaky = True,
language = "C++",
tags = [
"no_windows",

@ -74,7 +74,6 @@ grpc_cc_test(
":fling_client",
":fling_server",
],
flaky = True,
tags = [
"no_windows",
"requires-net:ipv4",

@ -27,7 +27,6 @@ grpc_cc_test(
"//src/core/tsi/test_creds:server1.pem",
],
external_deps = ["gtest"],
flaky = True,
language = "C++",
tags = ["no_windows"],
deps = [

@ -108,7 +108,6 @@ grpc_cc_test(
"//src/core/tsi/test_creds:server1.pem",
],
external_deps = ["gtest"],
flaky = True,
language = "C++",
tags = ["no_windows"],
deps = [

@ -349,7 +349,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
language = "C++",
tags = [
# TODO(apolcyn): This test is failing on Windows at entry, enable once passing.

@ -350,7 +350,6 @@ grpc_cc_test(
"//src/core/tsi/test_creds:server1.pem",
],
external_deps = ["gtest"],
flaky = True,
language = "C++",
deps = [
"//:gpr",
@ -399,7 +398,6 @@ grpc_cc_test(
"//src/core/tsi/test_creds:server1.pem",
],
external_deps = ["gtest"],
flaky = True,
language = "C++",
deps = [
"//:gpr",

@ -126,7 +126,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
tags = [
"no_test_android", # android_cc_test doesn't work with data dependency.
"no_test_ios",
@ -236,7 +235,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
deps = [
":interceptors_util",
":test_service_impl",
@ -691,7 +689,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
tags = [
"no_test_ios",
"no_windows",
@ -942,7 +939,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
deps = [
"//:grpc++",
"//:grpcpp_orca_service",

@ -225,7 +225,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
linkstatic = True, # Fixes dyld error on MacOS
tags = [
"no_test_ios",
@ -334,7 +333,6 @@ grpc_cc_test(
external_deps = [
"gtest",
],
flaky = True,
linkstatic = True, # Fixes dyld error on MacOS
tags = [
"no_test_ios",

@ -37,7 +37,6 @@ grpc_cc_test(
"gtest",
"opencensus-stats-test",
],
flaky = True,
tags = ["no_windows"],
deps = [
"//:gpr",

@ -59,6 +59,8 @@ client = bigquery.Client()
for row in client.query(
update_flakes_query.QUERY.format(
lookback_hours=lookback_hours)).result():
if "/macos/" in row.job_name:
continue # we know mac stuff is flaky
if row.test_binary not in tests:
m = re.match(r'^//test/core/end2end:([^@]*)@([^@]*)(.*)',
row.test_binary)

@ -137,6 +137,28 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": false,
"language": "c",
"name": "fling_stream_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -159,6 +181,30 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": false,
"language": "c",
"name": "goaway_server_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -1945,6 +1991,52 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "client_interceptors_end2end_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "client_ssl_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -2663,6 +2755,28 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "event_poller_posix_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -3371,6 +3485,30 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "grpc_tls_certificate_provider_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -3419,6 +3557,30 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "grpc_tls_credentials_options_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -3747,6 +3909,28 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "httpscli_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -4673,6 +4857,30 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix",
"windows"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "orca_service_end2end_test",
"platforms": [
"linux",
"mac",
"posix",
"windows"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -6359,6 +6567,28 @@
],
"uses_polling": false
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "stranded_event_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -7687,6 +7917,28 @@
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "xds_csds_end2end_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"benchmark": false,
@ -7711,6 +7963,28 @@
],
"uses_polling": false
},
{
"args": [],
"benchmark": false,
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": true,
"language": "c++",
"name": "xds_rls_end2end_test",
"platforms": [
"linux",
"mac",
"posix"
],
"uses_polling": true
},
{
"args": [],
"boringssl": true,

Loading…
Cancel
Save