diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 8a4092d83e6..26bab9c57b7 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -3458,7 +3458,6 @@ targets: - mac - name: fling_stream_test build: test - run: false language: c headers: - test/core/end2end/data/ssl_test_data.h @@ -3543,7 +3542,6 @@ targets: - mac - name: goaway_server_test build: test - run: false language: c headers: - test/core/end2end/cq_verifier.h @@ -5020,7 +5018,6 @@ targets: - name: client_interceptors_end2end_test gtest: true build: test - run: false language: c++ headers: - test/cpp/end2end/interceptors_util.h @@ -5066,7 +5063,6 @@ targets: - name: client_ssl_test gtest: true build: test - run: false language: c++ headers: [] src: @@ -5748,7 +5744,6 @@ targets: - name: event_poller_posix_test gtest: true build: test - run: false language: c++ headers: - src/core/lib/event_engine/common_closures.h @@ -6706,7 +6701,6 @@ targets: - name: grpc_tls_certificate_provider_test gtest: true build: test - run: false language: c++ headers: - test/core/util/cmdline.h @@ -6809,7 +6803,6 @@ targets: - name: grpc_tls_credentials_options_test gtest: true build: test - run: false language: c++ headers: - test/core/util/cmdline.h @@ -7170,7 +7163,6 @@ targets: - name: httpscli_test gtest: true build: test - run: false language: c++ headers: - test/core/http/httpcli_test_util.h @@ -7932,7 +7924,6 @@ targets: - name: orca_service_end2end_test gtest: true build: test - run: false language: c++ headers: [] src: @@ -9462,7 +9453,6 @@ targets: - name: stranded_event_test gtest: true build: test - run: false language: c++ headers: - test/core/end2end/cq_verifier.h @@ -10957,7 +10947,6 @@ targets: - name: xds_csds_end2end_test gtest: true build: test - run: false language: c++ headers: - src/cpp/server/csds/csds.h @@ -11343,7 +11332,6 @@ targets: - name: xds_rls_end2end_test gtest: true build: test - run: false language: c++ headers: - test/cpp/end2end/counted_service.h diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index bcc466598ac..237a5d7d8e1 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -168,7 +168,6 @@ grpc_cc_test( "absl/status:statusor", "absl/strings", ], - flaky = True, language = "C++", deps = [ "cq_verifier", diff --git a/test/core/end2end/flaky.bzl b/test/core/end2end/flaky.bzl index 01e88cda12d..49ac1ca203c 100644 --- a/test/core/end2end/flaky.bzl +++ b/test/core/end2end/flaky.bzl @@ -13,181 +13,50 @@ # limitations under the License. """A list of flaky tests, consumed by generate_tests.bzl to set flaky attrs.""" FLAKY_TESTS = [ - "h2_census_test@grpc_authz", - "h2_census_test@retry_server_pushback_delay", - "h2_census_test@retry_streaming2", - "h2_census_test@retry_too_many_attempts", - "h2_census_test@retry_transparent_mcs", - "h2_census_test@shutdown_finishes_calls", - "h2_compress_test@bad_ping", - "h2_compress_test@grpc_authz", "h2_compress_test@max_connection_age", - "h2_compress_test@max_connection_idle", - "h2_compress_test@retry_cancellation", - "h2_compress_test@retry_per_attempt_recv_timeout", - "h2_fakesec_test@bad_ping", - "h2_fakesec_test@max_connection_idle", - "h2_fakesec_test@retry_per_attempt_recv_timeout", - "h2_fakesec_test@retry_server_pushback_disabled", - "h2_fakesec_test@retry_too_many_attempts", - "h2_fakesec_test@shutdown_finishes_calls", - "h2_fd_test@filtered_metadata", - "h2_fd_test@max_connection_age", "h2_full+pipe_test@max_connection_age", - "h2_full+pipe_test@max_connection_idle", - "h2_full_no_retry_test@bad_ping", - "h2_full_no_retry_test@grpc_authz", - "h2_full_no_retry_test@hpack_size", "h2_full_no_retry_test@max_connection_age", - "h2_full_no_retry_test@max_connection_idle", - "h2_full_test@bad_ping", - "h2_full_test@keepalive_timeout", "h2_full_test@max_connection_age", "h2_full_test@max_connection_idle", - "h2_full_test@retry_per_attempt_recv_timeout", - "h2_full_test@retry_recv_trailing_metadata_error", - "h2_full_test@retry_server_pushback_delay", - "h2_full_test@retry_transparent_mcs", - "h2_http_proxy_test@compressed_payload", - "h2_http_proxy_test@connectivity", - "h2_http_proxy_test@grpc_authz", - "h2_http_proxy_test@hpack_size", "h2_http_proxy_test@max_connection_age", - "h2_http_proxy_test@max_connection_idle", - "h2_http_proxy_test@ping_pong_streaming", "h2_http_proxy_test@retry_per_attempt_recv_timeout", - "h2_http_proxy_test@retry_server_pushback_disabled", - "h2_insecure_test@bad_ping", - "h2_insecure_test@grpc_authz", - "h2_insecure_test@max_connection_age", "h2_insecure_test@max_connection_idle", - "h2_insecure_test@retry", - "h2_insecure_test@retry_server_pushback_disabled", - "h2_insecure_test@retry_streaming", "h2_local_abstract_uds_percent_encoded_test@max_connection_idle", - "h2_local_ipv4_test@bad_ping", "h2_local_ipv4_test@max_connection_idle", - "h2_local_ipv4_test@retry", - "h2_local_ipv4_test@retry_per_attempt_recv_timeout", - "h2_local_ipv4_test@retry_server_pushback_delay", - "h2_local_ipv6_test@cancel_after_accept", - "h2_local_ipv6_test@grpc_authz", "h2_local_ipv6_test@max_connection_idle", - "h2_local_ipv6_test@retry_per_attempt_recv_timeout", - "h2_local_ipv6_test@retry_send_initial_metadata_refs", - "h2_local_ipv6_test@retry_server_pushback_disabled", - "h2_local_ipv6_test@shutdown_finishes_calls", - "h2_local_uds_percent_encoded_test@bad_ping", - "h2_local_uds_percent_encoded_test@connectivity", - "h2_local_uds_percent_encoded_test@grpc_authz", "h2_local_uds_percent_encoded_test@max_connection_idle", - "h2_local_uds_percent_encoded_test@ping", - "h2_local_uds_percent_encoded_test@retry_per_attempt_recv_timeout", - "h2_local_uds_percent_encoded_test@retry_server_pushback_delay", - "h2_local_uds_percent_encoded_test@retry_streaming2", - "h2_local_uds_percent_encoded_test@retry_transparent_mcs", - "h2_local_uds_percent_encoded_test@shutdown_finishes_calls", - "h2_local_uds_test@bad_ping", - "h2_local_uds_test@grpc_authz", - "h2_local_uds_test@max_connection_age", "h2_local_uds_test@max_connection_idle", - "h2_local_uds_test@retry_per_attempt_recv_timeout", - "h2_local_uds_test@retry_server_pushback_delay", - "h2_local_uds_test@retry_server_pushback_disabled", - "h2_oauth2_test@bad_ping", - "h2_oauth2_test@connectivity", - "h2_oauth2_test@filter_context", - "h2_oauth2_test@grpc_authz", - "h2_oauth2_test@max_connection_age", "h2_oauth2_test@max_connection_idle", - "h2_oauth2_test@retry_too_many_attempts", "h2_oauth2_tls12_test@max_connection_idle", - "h2_oauth2_tls12_test@retry_per_attempt_recv_timeout", - "h2_oauth2_tls12_test@retry_server_pushback_delay", - "h2_oauth2_tls13_test@bad_ping", - "h2_proxy_test@grpc_authz", - "h2_proxy_test@retry_send_initial_metadata_refs", - "h2_proxy_test@retry_streaming2", - "h2_proxy_test@shutdown_finishes_calls", + "h2_proxy_test@cancel_with_status", "h2_sockpair_1byte_test@max_connection_age", - "h2_sockpair_1byte_test@shutdown_finishes_calls", - "h2_sockpair_test@max_connection_age", - "h2_ssl_cred_reload_test@bad_ping", - "h2_ssl_cred_reload_test@client_streaming", - "h2_ssl_cred_reload_test@connectivity", "h2_ssl_cred_reload_test@max_connection_idle", - "h2_ssl_cred_reload_test@retry", - "h2_ssl_cred_reload_test@retry_per_attempt_recv_timeout", - "h2_ssl_cred_reload_test@retry_server_pushback_delay", - "h2_ssl_cred_reload_test@simple_delayed_request", - "h2_ssl_cred_reload_tls12_test@bad_ping", "h2_ssl_cred_reload_tls12_test@max_connection_age", - "h2_ssl_cred_reload_tls12_test@max_connection_idle", - "h2_ssl_cred_reload_tls12_test@retry_server_pushback_delay", - "h2_ssl_cred_reload_tls12_test@retry_too_many_attempts", - "h2_ssl_cred_reload_tls12_test@shutdown_finishes_calls", - "h2_ssl_cred_reload_tls13_test@retry_per_attempt_recv_timeout", - "h2_ssl_proxy_test@retry_exceeds_buffer_size_in_delay", - "h2_ssl_proxy_test@retry_server_pushback_delay", - "h2_ssl_proxy_test@shutdown_finishes_calls", - "h2_ssl_test@bad_ping", - "h2_ssl_test@compressed_payload", + "h2_ssl_proxy_test@retry_transparent_not_sent_on_wire", "h2_ssl_test@max_connection_idle", "h2_ssl_test@retry_cancellation", - "h2_ssl_test@retry_per_attempt_recv_timeout", - "h2_ssl_test@retry_send_initial_metadata_refs", - "h2_ssl_test@retry_server_pushback_disabled", - "h2_ssl_test@retry_too_many_attempts", - "h2_ssl_test@shutdown_finishes_calls", - "h2_ssl_tls12_test@client_streaming", - "h2_ssl_tls12_test@max_connection_idle", - "h2_ssl_tls13_test@retry_per_attempt_recv_timeout", - "h2_tls_certwatch_async_tls1_3_test@cancel_after_invoke", "h2_tls_certwatch_async_tls1_3_test@connectivity", "h2_tls_certwatch_async_tls1_3_test@max_connection_idle", "h2_tls_certwatch_async_tls1_3_test@retry_cancellation", - "h2_tls_certwatch_async_tls1_3_test@retry_streaming", - "h2_tls_certwatch_async_tls1_3_test@retry_streaming2", "h2_tls_certwatch_async_tls1_3_test@simple_delayed_request", "h2_tls_certwatch_sync_tls1_2_test@connectivity", "h2_tls_certwatch_sync_tls1_2_test@max_connection_idle", "h2_tls_certwatch_sync_tls1_2_test@retry_cancellation", - "h2_tls_certwatch_sync_tls1_2_test@retry_server_pushback_delay", "h2_tls_certwatch_sync_tls1_2_test@simple_delayed_request", - "h2_tls_certwatch_sync_tls1_2_test@write_buffering", - "h2_tls_simple_test@bad_ping", "h2_tls_simple_test@connectivity", - "h2_tls_simple_test@grpc_authz", "h2_tls_simple_test@max_connection_idle", - "h2_tls_simple_test@retry_per_attempt_recv_timeout", "h2_tls_simple_test@simple_delayed_request", - "h2_tls_simple_test@write_buffering", "h2_tls_static_async_tls1_3_test@connectivity", "h2_tls_static_async_tls1_3_test@max_connection_idle", - "h2_tls_static_async_tls1_3_test@ping", "h2_tls_static_async_tls1_3_test@retry_cancellation", - "h2_tls_static_async_tls1_3_test@retry_per_attempt_recv_timeout", - "h2_tls_static_async_tls1_3_test@retry_server_pushback_delay", - "h2_tls_static_async_tls1_3_test@retry_streaming", "h2_tls_static_async_tls1_3_test@simple_delayed_request", - "h2_tls_test@bad_ping", "h2_tls_test@connectivity", - "h2_tls_test@grpc_authz", "h2_tls_test@hpack_size", "h2_tls_test@invoke_large_request", - "h2_tls_test@keepalive_timeout", "h2_tls_test@max_connection_idle", "h2_tls_test@resource_quota_server", - "h2_tls_test@retry", "h2_tls_test@retry_cancellation", - "h2_tls_test@retry_per_attempt_recv_timeout", - "h2_tls_test@retry_streaming2", "h2_tls_test@simple_delayed_request", "h2_uds_abstract_test@max_connection_age", - "h2_uds_test@bad_ping", "h2_uds_test@max_connection_idle", - "h2_uds_test@retry_per_attempt_recv_timeout", - "h2_uds_test@retry_server_pushback_delay", - "h2_uds_test@retry_streaming", - "h2_uds_test@retry_transparent_mcs", ] diff --git a/test/core/event_engine/posix/BUILD b/test/core/event_engine/posix/BUILD index c5871a05963..4d5a94ef997 100644 --- a/test/core/event_engine/posix/BUILD +++ b/test/core/event_engine/posix/BUILD @@ -65,7 +65,6 @@ grpc_cc_test( name = "event_poller_posix_test", srcs = ["event_poller_posix_test.cc"], external_deps = ["gtest"], - flaky = True, language = "C++", tags = [ "no_windows", diff --git a/test/core/fling/BUILD b/test/core/fling/BUILD index 4894bfdb3ac..e4fe3a37303 100644 --- a/test/core/fling/BUILD +++ b/test/core/fling/BUILD @@ -74,7 +74,6 @@ grpc_cc_test( ":fling_client", ":fling_server", ], - flaky = True, tags = [ "no_windows", "requires-net:ipv4", diff --git a/test/core/handshake/BUILD b/test/core/handshake/BUILD index fcceba53607..291a6e05e1f 100644 --- a/test/core/handshake/BUILD +++ b/test/core/handshake/BUILD @@ -27,7 +27,6 @@ grpc_cc_test( "//src/core/tsi/test_creds:server1.pem", ], external_deps = ["gtest"], - flaky = True, language = "C++", tags = ["no_windows"], deps = [ diff --git a/test/core/http/BUILD b/test/core/http/BUILD index 5733bd0215b..f2a5d7c7190 100644 --- a/test/core/http/BUILD +++ b/test/core/http/BUILD @@ -108,7 +108,6 @@ grpc_cc_test( "//src/core/tsi/test_creds:server1.pem", ], external_deps = ["gtest"], - flaky = True, language = "C++", tags = ["no_windows"], deps = [ diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index 7bc51281b09..e76db2e8f82 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -349,7 +349,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, language = "C++", tags = [ # TODO(apolcyn): This test is failing on Windows at entry, enable once passing. diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 699600d1661..2a614b15d32 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -350,7 +350,6 @@ grpc_cc_test( "//src/core/tsi/test_creds:server1.pem", ], external_deps = ["gtest"], - flaky = True, language = "C++", deps = [ "//:gpr", @@ -399,7 +398,6 @@ grpc_cc_test( "//src/core/tsi/test_creds:server1.pem", ], external_deps = ["gtest"], - flaky = True, language = "C++", deps = [ "//:gpr", diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index beb15bb7750..7f06cb63168 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -126,7 +126,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, tags = [ "no_test_android", # android_cc_test doesn't work with data dependency. "no_test_ios", @@ -236,7 +235,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, deps = [ ":interceptors_util", ":test_service_impl", @@ -691,7 +689,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, tags = [ "no_test_ios", "no_windows", @@ -942,7 +939,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, deps = [ "//:grpc++", "//:grpcpp_orca_service", diff --git a/test/cpp/end2end/xds/BUILD b/test/cpp/end2end/xds/BUILD index b117d771dec..9d7765380dd 100644 --- a/test/cpp/end2end/xds/BUILD +++ b/test/cpp/end2end/xds/BUILD @@ -225,7 +225,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, linkstatic = True, # Fixes dyld error on MacOS tags = [ "no_test_ios", @@ -334,7 +333,6 @@ grpc_cc_test( external_deps = [ "gtest", ], - flaky = True, linkstatic = True, # Fixes dyld error on MacOS tags = [ "no_test_ios", diff --git a/test/cpp/server/load_reporter/BUILD b/test/cpp/server/load_reporter/BUILD index 850ab36847c..2fd5da92472 100644 --- a/test/cpp/server/load_reporter/BUILD +++ b/test/cpp/server/load_reporter/BUILD @@ -37,7 +37,6 @@ grpc_cc_test( "gtest", "opencensus-stats-test", ], - flaky = True, tags = ["no_windows"], deps = [ "//:gpr", diff --git a/tools/distrib/update_flakes.py b/tools/distrib/update_flakes.py index 9bdfb6986d0..16e8304e836 100755 --- a/tools/distrib/update_flakes.py +++ b/tools/distrib/update_flakes.py @@ -59,6 +59,8 @@ client = bigquery.Client() for row in client.query( update_flakes_query.QUERY.format( lookback_hours=lookback_hours)).result(): + if "/macos/" in row.job_name: + continue # we know mac stuff is flaky if row.test_binary not in tests: m = re.match(r'^//test/core/end2end:([^@]*)@([^@]*)(.*)', row.test_binary) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 9ae6e0597db..e85edc2a602 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -137,6 +137,28 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "fling_stream_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -159,6 +181,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "goaway_server_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -1945,6 +1991,52 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "client_interceptors_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "client_ssl_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -2663,6 +2755,28 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "event_poller_posix_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -3371,6 +3485,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_tls_certificate_provider_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -3419,6 +3557,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_tls_credentials_options_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -3747,6 +3909,28 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "httpscli_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -4673,6 +4857,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "orca_service_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -6359,6 +6567,28 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "stranded_event_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -7687,6 +7917,28 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "xds_csds_end2end_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -7711,6 +7963,28 @@ ], "uses_polling": false }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "xds_rls_end2end_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "boringssl": true,