[experiments] Assume enabling experiments is flaky (#33116)

This will change behavior for tests that have experiments enabled on
them to always have the flaky bit on.

In doing so, we'll get the usual failure reporting we do in the internal
chat bot, but allow PRs to pass even if an experiment isn't 100% passing
yet - reducing friction slightly for landing bigger experiments.

<!--

If you know who should review your pull request, please assign it to
that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the
appropriate
lang label.

-->

bazel/grpc_build_system.bzl

@@ -269,7 +269,7 @@ def ios_cc_test(
             deps = ios_test_deps,
         )
 
-def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, uses_event_engine):
+def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, uses_event_engine, flaky):
     """Common logic used to parameterize tests for every poller and EventEngine and experiment.
 
     Args:
@@ -278,6 +278,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
         deps: base deps
         tags: base tags
         args: base args
+        flaky: base flaky
         exclude_pollers: list of poller names to exclude for this set of tests.
         uses_polling: set to False if the test is not sensitive to polling methodology.
         uses_event_engine: set to False if the test is not sensitive to
@@ -297,6 +298,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
             "deps": deps,
             "tags": tags,
             "args": args,
+            "flaky": flaky,
             "env": {},
         })
     else:
@@ -318,6 +320,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
                 "env": {
                     "GRPC_POLL_STRATEGY": poller,
                 },
+                "flaky": flaky,
             })
 
         # Now generate one test for each subsequent EventEngine, all using the
@@ -334,6 +337,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
                 "tags": tags + ["no_linux"],
                 "args": args,
                 "env": {},
+                "flaky": flaky,
             })
         else:
             for engine_name, engine in EVENT_ENGINES.items():
@@ -353,6 +357,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
                     "tags": test_tags,
                     "args": test_args,
                     "env": {},
+                    "flaky": flaky,
                 })
 
     experiments = {}
@@ -399,6 +404,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
                         if tag not in tags:
                             tags = tags + [tag]
                     config["tags"] = tags
+                    config["flaky"] = True
                     experiment_config.append(config)
         if disabled_tags != None:
             for experiment in experiments[mode]:
@@ -458,7 +464,6 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
         "exec_compatible_with": exec_compatible_with,
         "exec_properties": exec_properties,
         "shard_count": shard_count,
-        "flaky": flaky,
         "linkstatic": linkstatic,
     }
 
@@ -469,10 +474,11 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
             tags = tags,
             deps = core_deps,
             args = args,
+            flaky = flaky,
             **test_args
         )
 
-    for poller_config in expand_tests(name, srcs, core_deps, tags, args, exclude_pollers, uses_polling, uses_event_engine):
+    for poller_config in expand_tests(name, srcs, core_deps, tags, args, exclude_pollers, uses_polling, uses_event_engine, flaky):
         native.cc_test(
             name = poller_config["name"],
             srcs = poller_config["srcs"],
@@ -480,6 +486,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
             tags = poller_config["tags"],
             args = poller_config["args"],
             env = poller_config["env"],
+            flaky = poller_config["flaky"],
             **test_args
         )
 
@@ -563,10 +570,9 @@ def grpc_sh_test(name, srcs = [], args = [], data = [], uses_polling = True, siz
         "exec_compatible_with": exec_compatible_with,
         "exec_properties": exec_properties,
         "shard_count": shard_count,
-        "flaky": flaky,
     }
 
-    for poller_config in expand_tests(name, srcs, [], tags, args, exclude_pollers, uses_polling, uses_event_engine):
+    for poller_config in expand_tests(name, srcs, [], tags, args, exclude_pollers, uses_polling, uses_event_engine, flaky):
         native.sh_test(
             name = poller_config["name"],
             srcs = poller_config["srcs"],
@@ -574,6 +580,7 @@ def grpc_sh_test(name, srcs = [], args = [], data = [], uses_polling = True, siz
             tags = poller_config["tags"],
             args = poller_config["args"],
             env = poller_config["env"],
+            flaky = poller_config["flaky"],
             **test_args
         )