[e2e] Mark a few tests flaky (#37880)

I'm continuing to look into some flakes here, but in the meantime these shouldn't halt submissions. Marking them flaky.

Closes #37880

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37880 from ctiller:mark-flaky 27427c7978
PiperOrigin-RevId: 684526341

build_autogenerated.yaml

@@ -7481,6 +7481,7 @@ targets:
 - name: cancel_after_invoke_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers:
   - src/core/ext/transport/chaotic_good/chaotic_good_transport.h
@@ -7858,6 +7859,7 @@ targets:
 - name: cancel_with_status_test
   gtest: true
   build: test
+  run: false
   language: c++
   headers:
   - src/core/ext/transport/chaotic_good/chaotic_good_transport.h

test/core/end2end/BUILD

@@ -312,6 +312,7 @@ grpc_core_end2end_test(
 
 grpc_core_end2end_test(
     name = "cancel_after_invoke",
+    flaky = True,
     shard_count = 50,
 )
 
@@ -324,7 +325,10 @@ grpc_core_end2end_test(name = "cancel_before_invoke")
 
 grpc_core_end2end_test(name = "cancel_in_a_vacuum")
 
-grpc_core_end2end_test(name = "cancel_with_status")
+grpc_core_end2end_test(
+    name = "cancel_with_status",
+    flaky = True,
+)
 
 grpc_core_end2end_test(name = "channelz")
 

test/core/end2end/grpc_core_end2end_test.bzl

@@ -25,13 +25,14 @@ END2END_TEST_DATA = [
     "//src/core/tsi/test_creds:server1.pem",
 ]
 
-def grpc_core_end2end_test(name, shard_count = 10, tags = []):
+def grpc_core_end2end_test(name, shard_count = 10, tags = [], flaky = False):
     """Generate one core end2end test
 
     Args:
         name: name of the test, must correspond to a "test/name.cc" file
         shard_count: per bazel
         tags: per bazel
+        flaky: per bazel
     """
 
     if len(name) > 60:
@@ -112,6 +113,7 @@ def grpc_core_end2end_test(name, shard_count = 10, tags = []):
             "%s_library" % name,
         ],
         tags = ["core_end2end_test", "thready_tsan"] + tags,
+        flaky = flaky,
     )
 
     grpc_proto_fuzzer(

tools/run_tests/generated/tests.json

@@ -1515,30 +1515,6 @@
     ],
     "uses_polling": true
   },
-  {
-    "args": [],
-    "benchmark": false,
-    "ci_platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "cpu_cost": 1.0,
-    "exclude_configs": [],
-    "exclude_iomgrs": [],
-    "flaky": false,
-    "gtest": true,
-    "language": "c++",
-    "name": "cancel_after_invoke_test",
-    "platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "uses_polling": true
-  },
   {
     "args": [],
     "benchmark": false,
@@ -1659,30 +1635,6 @@
     ],
     "uses_polling": true
   },
-  {
-    "args": [],
-    "benchmark": false,
-    "ci_platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "cpu_cost": 1.0,
-    "exclude_configs": [],
-    "exclude_iomgrs": [],
-    "flaky": false,
-    "gtest": true,
-    "language": "c++",
-    "name": "cancel_with_status_test",
-    "platforms": [
-      "linux",
-      "mac",
-      "posix",
-      "windows"
-    ],
-    "uses_polling": true
-  },
   {
     "args": [],
     "benchmark": false,