Update flaky tests list (#30771)

* Update flaky tests list

* Automated change: Fix sanity tests

Co-authored-by: ctiller <ctiller@users.noreply.github.com>

build_autogenerated.yaml

@@ -3531,6 +3531,7 @@ targets:
   - mac
 - name: goaway_server_test
   build: test
+  run: false
   language: c
   headers:
   - test/core/end2end/cq_verifier.h
@@ -11182,6 +11183,7 @@ targets:
 - name: xds_csds_end2end_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers:
   - src/cpp/server/csds/csds.h

test/core/end2end/BUILD

@@ -168,6 +168,7 @@ grpc_cc_test(
         "absl/status:statusor",
         "absl/strings",
     ],
+    flaky = True,
     language = "C++",
     deps = [
         "cq_verifier",

test/core/end2end/flaky.bzl

@@ -13,24 +13,27 @@
 # limitations under the License.
 """A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs."""
 FLAKY_TESTS = [
-    "h2_census_test@bad_ping",
     "h2_census_test@grpc_authz",
-    "h2_census_test@retry_per_attempt_recv_timeout",
+    "h2_census_test@retry_server_pushback_delay",
+    "h2_census_test@retry_streaming2",
     "h2_census_test@retry_too_many_attempts",
     "h2_census_test@retry_transparent_mcs",
     "h2_census_test@shutdown_finishes_calls",
+    "h2_compress_test@bad_ping",
     "h2_compress_test@grpc_authz",
-    "h2_compress_test@keepalive_timeout",
     "h2_compress_test@max_connection_age",
     "h2_compress_test@max_connection_idle",
     "h2_compress_test@retry_cancellation",
     "h2_compress_test@retry_per_attempt_recv_timeout",
+    "h2_fakesec_test@bad_ping",
     "h2_fakesec_test@max_connection_idle",
     "h2_fakesec_test@retry_per_attempt_recv_timeout",
     "h2_fakesec_test@retry_server_pushback_disabled",
     "h2_fakesec_test@retry_too_many_attempts",
     "h2_fakesec_test@shutdown_finishes_calls",
+    "h2_fd_test@filtered_metadata",
     "h2_fd_test@max_connection_age",
+    "h2_full+pipe_test@max_connection_age",
     "h2_full+pipe_test@max_connection_idle",
     "h2_full_no_retry_test@bad_ping",
     "h2_full_no_retry_test@grpc_authz",
@@ -43,8 +46,8 @@ FLAKY_TESTS = [
     "h2_full_test@max_connection_idle",
     "h2_full_test@retry_per_attempt_recv_timeout",
     "h2_full_test@retry_recv_trailing_metadata_error",
-    "h2_full_test@retry_send_initial_metadata_refs",
     "h2_full_test@retry_server_pushback_delay",
+    "h2_full_test@retry_transparent_mcs",
     "h2_http_proxy_test@compressed_payload",
     "h2_http_proxy_test@connectivity",
     "h2_http_proxy_test@grpc_authz",
@@ -53,36 +56,43 @@ FLAKY_TESTS = [
     "h2_http_proxy_test@max_connection_idle",
     "h2_http_proxy_test@ping_pong_streaming",
     "h2_http_proxy_test@retry_per_attempt_recv_timeout",
+    "h2_http_proxy_test@retry_server_pushback_disabled",
     "h2_insecure_test@bad_ping",
     "h2_insecure_test@grpc_authz",
     "h2_insecure_test@max_connection_age",
     "h2_insecure_test@max_connection_idle",
     "h2_insecure_test@retry",
     "h2_insecure_test@retry_server_pushback_disabled",
+    "h2_insecure_test@retry_streaming",
     "h2_local_abstract_uds_percent_encoded_test@max_connection_idle",
     "h2_local_ipv4_test@bad_ping",
     "h2_local_ipv4_test@max_connection_idle",
     "h2_local_ipv4_test@retry",
     "h2_local_ipv4_test@retry_per_attempt_recv_timeout",
     "h2_local_ipv4_test@retry_server_pushback_delay",
-    "h2_local_ipv4_test@retry_streaming2",
-    "h2_local_ipv6_test@bad_ping",
     "h2_local_ipv6_test@cancel_after_accept",
+    "h2_local_ipv6_test@grpc_authz",
     "h2_local_ipv6_test@max_connection_idle",
     "h2_local_ipv6_test@retry_per_attempt_recv_timeout",
+    "h2_local_ipv6_test@retry_send_initial_metadata_refs",
     "h2_local_ipv6_test@retry_server_pushback_disabled",
     "h2_local_ipv6_test@shutdown_finishes_calls",
     "h2_local_uds_percent_encoded_test@bad_ping",
     "h2_local_uds_percent_encoded_test@connectivity",
+    "h2_local_uds_percent_encoded_test@grpc_authz",
     "h2_local_uds_percent_encoded_test@max_connection_idle",
     "h2_local_uds_percent_encoded_test@ping",
     "h2_local_uds_percent_encoded_test@retry_per_attempt_recv_timeout",
+    "h2_local_uds_percent_encoded_test@retry_server_pushback_delay",
     "h2_local_uds_percent_encoded_test@retry_streaming2",
     "h2_local_uds_percent_encoded_test@retry_transparent_mcs",
+    "h2_local_uds_percent_encoded_test@shutdown_finishes_calls",
     "h2_local_uds_test@bad_ping",
     "h2_local_uds_test@grpc_authz",
     "h2_local_uds_test@max_connection_age",
     "h2_local_uds_test@max_connection_idle",
+    "h2_local_uds_test@retry_per_attempt_recv_timeout",
+    "h2_local_uds_test@retry_server_pushback_delay",
     "h2_local_uds_test@retry_server_pushback_disabled",
     "h2_oauth2_test@bad_ping",
     "h2_oauth2_test@connectivity",
@@ -90,14 +100,17 @@ FLAKY_TESTS = [
     "h2_oauth2_test@grpc_authz",
     "h2_oauth2_test@max_connection_age",
     "h2_oauth2_test@max_connection_idle",
-    "h2_oauth2_test@retry_server_pushback_disabled",
     "h2_oauth2_test@retry_too_many_attempts",
+    "h2_oauth2_tls12_test@max_connection_idle",
+    "h2_oauth2_tls12_test@retry_per_attempt_recv_timeout",
+    "h2_oauth2_tls12_test@retry_server_pushback_delay",
+    "h2_oauth2_tls13_test@bad_ping",
     "h2_proxy_test@grpc_authz",
     "h2_proxy_test@retry_send_initial_metadata_refs",
+    "h2_proxy_test@retry_streaming2",
     "h2_proxy_test@shutdown_finishes_calls",
     "h2_sockpair_1byte_test@max_connection_age",
     "h2_sockpair_1byte_test@shutdown_finishes_calls",
-    "h2_sockpair_test@grpc_authz",
     "h2_sockpair_test@max_connection_age",
     "h2_ssl_cred_reload_test@bad_ping",
     "h2_ssl_cred_reload_test@client_streaming",
@@ -106,25 +119,29 @@ FLAKY_TESTS = [
     "h2_ssl_cred_reload_test@retry",
     "h2_ssl_cred_reload_test@retry_per_attempt_recv_timeout",
     "h2_ssl_cred_reload_test@retry_server_pushback_delay",
-    "h2_ssl_cred_reload_test@retry_too_many_attempts",
     "h2_ssl_cred_reload_test@simple_delayed_request",
+    "h2_ssl_cred_reload_tls12_test@bad_ping",
+    "h2_ssl_cred_reload_tls12_test@max_connection_age",
+    "h2_ssl_cred_reload_tls12_test@max_connection_idle",
     "h2_ssl_cred_reload_tls12_test@retry_server_pushback_delay",
     "h2_ssl_cred_reload_tls12_test@retry_too_many_attempts",
-    "h2_ssl_proxy_test@disappearing_server",
+    "h2_ssl_cred_reload_tls12_test@shutdown_finishes_calls",
+    "h2_ssl_cred_reload_tls13_test@retry_per_attempt_recv_timeout",
     "h2_ssl_proxy_test@retry_exceeds_buffer_size_in_delay",
     "h2_ssl_proxy_test@retry_server_pushback_delay",
     "h2_ssl_proxy_test@shutdown_finishes_calls",
     "h2_ssl_test@bad_ping",
     "h2_ssl_test@compressed_payload",
-    "h2_ssl_test@grpc_authz",
     "h2_ssl_test@max_connection_idle",
-    "h2_ssl_test@retry",
     "h2_ssl_test@retry_cancellation",
     "h2_ssl_test@retry_per_attempt_recv_timeout",
     "h2_ssl_test@retry_send_initial_metadata_refs",
     "h2_ssl_test@retry_server_pushback_disabled",
     "h2_ssl_test@retry_too_many_attempts",
     "h2_ssl_test@shutdown_finishes_calls",
+    "h2_ssl_tls12_test@client_streaming",
+    "h2_ssl_tls12_test@max_connection_idle",
+    "h2_ssl_tls13_test@retry_per_attempt_recv_timeout",
     "h2_tls_certwatch_async_tls1_3_test@cancel_after_invoke",
     "h2_tls_certwatch_async_tls1_3_test@connectivity",
     "h2_tls_certwatch_async_tls1_3_test@max_connection_idle",
@@ -140,14 +157,18 @@ FLAKY_TESTS = [
     "h2_tls_certwatch_sync_tls1_2_test@write_buffering",
     "h2_tls_simple_test@bad_ping",
     "h2_tls_simple_test@connectivity",
+    "h2_tls_simple_test@grpc_authz",
     "h2_tls_simple_test@max_connection_idle",
     "h2_tls_simple_test@retry_per_attempt_recv_timeout",
     "h2_tls_simple_test@simple_delayed_request",
+    "h2_tls_simple_test@write_buffering",
     "h2_tls_static_async_tls1_3_test@connectivity",
     "h2_tls_static_async_tls1_3_test@max_connection_idle",
+    "h2_tls_static_async_tls1_3_test@ping",
     "h2_tls_static_async_tls1_3_test@retry_cancellation",
     "h2_tls_static_async_tls1_3_test@retry_per_attempt_recv_timeout",
     "h2_tls_static_async_tls1_3_test@retry_server_pushback_delay",
+    "h2_tls_static_async_tls1_3_test@retry_streaming",
     "h2_tls_static_async_tls1_3_test@simple_delayed_request",
     "h2_tls_test@bad_ping",
     "h2_tls_test@connectivity",
@@ -160,12 +181,13 @@ FLAKY_TESTS = [
     "h2_tls_test@retry",
     "h2_tls_test@retry_cancellation",
     "h2_tls_test@retry_per_attempt_recv_timeout",
-    "h2_tls_test@retry_streaming",
     "h2_tls_test@retry_streaming2",
     "h2_tls_test@simple_delayed_request",
+    "h2_uds_abstract_test@max_connection_age",
     "h2_uds_test@bad_ping",
     "h2_uds_test@max_connection_idle",
     "h2_uds_test@retry_per_attempt_recv_timeout",
+    "h2_uds_test@retry_server_pushback_delay",
     "h2_uds_test@retry_streaming",
     "h2_uds_test@retry_transparent_mcs",
 ]

test/core/memory_usage/BUILD

@@ -151,7 +151,6 @@ grpc_cc_test(
         "absl/flags:flag",
         "absl/flags:parse",
     ],
-    flaky = True,
     language = "C++",
     tags = [
         "bazel_only",

test/cpp/end2end/xds/BUILD

@@ -225,6 +225,7 @@ grpc_cc_test(
     external_deps = [
         "gtest",
     ],
+    flaky = True,
     linkstatic = True,  # Fixes dyld error on MacOS
     tags = [
         "no_test_ios",

test/cpp/microbenchmarks/BUILD

@@ -251,7 +251,6 @@ grpc_cc_test(
         "bm_fullstack_unary_ping_pong.cc",
     ],
     args = grpc_benchmark_args(),
-    flaky = True,
     tags = [
         "no_mac",  # to emulate "excluded_poll_engines: poll"
         "no_windows",

tools/run_tests/generated/tests.json

@@ -159,30 +159,6 @@
     ],
     "uses_polling": true
   },
-  {
-    "args": [],
-    "benchmark": false,
-    "ci_platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "cpu_cost": 1.0,
-    "exclude_configs": [],
-    "exclude_iomgrs": [],
-    "flaky": false,
-    "gtest": false,
-    "language": "c",
-    "name": "goaway_server_test",
-    "platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "uses_polling": true
-  },
   {
     "args": [],
     "benchmark": false,
@@ -7687,28 +7663,6 @@
     ],
     "uses_polling": true
   },
-  {
-    "args": [],
-    "benchmark": false,
-    "ci_platforms": [
-      "linux",
-      "mac",
-      "posix"
-    ],
-    "cpu_cost": 1.0,
-    "exclude_configs": [],
-    "exclude_iomgrs": [],
-    "flaky": false,
-    "gtest": true,
-    "language": "c++",
-    "name": "xds_csds_end2end_test",
-    "platforms": [
-      "linux",
-      "mac",
-      "posix"
-    ],
-    "uses_polling": true
-  },
   {
     "args": [],
     "benchmark": false,