[fuzzing] Allow core_end2end_test_fuzzer, api_fuzzer to change experiments (#33147)

They were intended to be able to, but since these are currently frozen
across the process it wasn't possible.

Fix that for these fuzzers.

src/core/lib/experiments/config.cc

@@ -48,10 +48,10 @@ struct ForcedExperiment {
 };
 ForcedExperiment g_forced_experiments[kNumExperiments];
 
-std::atomic<bool> g_loaded;
+std::atomic<bool> g_loaded(false);
 
 GPR_ATTRIBUTE_NOINLINE Experiments LoadExperimentsFromConfigVariable() {
-  GPR_ASSERT(g_loaded.exchange(true, std::memory_order_relaxed) == false);
+  g_loaded.store(true, std::memory_order_relaxed);
   // Set defaults from metadata.
   Experiments experiments;
   for (size_t i = 0; i < kNumExperiments; i++) {
@@ -92,14 +92,23 @@ GPR_ATTRIBUTE_NOINLINE Experiments LoadExperimentsFromConfigVariable() {
   }
   return experiments;
 }
-}  // namespace
 
-bool IsExperimentEnabled(size_t experiment_id) {
+Experiments& ExperimentsSingleton() {
   // One time initialization:
-  static const NoDestruct<Experiments> experiments{
+  static NoDestruct<Experiments> experiments{
       LoadExperimentsFromConfigVariable()};
+  return *experiments;
+}
+}  // namespace
+
+void TestOnlyReloadExperimentsFromConfigVariables() {
+  ExperimentsSingleton() = LoadExperimentsFromConfigVariable();
+  PrintExperimentsList();
+}
+
+bool IsExperimentEnabled(size_t experiment_id) {
   // Normal path: just return the value;
-  return experiments->enabled[experiment_id];
+  return ExperimentsSingleton().enabled[experiment_id];
 }
 
 void PrintExperimentsList() {

src/core/lib/experiments/config.h

@@ -30,6 +30,12 @@ namespace grpc_core {
 // Experiments are numbered by their order in the g_experiment_metadata array
 // declared in experiments.h.
 bool IsExperimentEnabled(size_t experiment_id);
+
+// Reload experiment state from config variables.
+// Does not change ForceEnableExperiment state.
+// Expects the caller to handle global thread safety - so really only
+// appropriate for carefully written tests.
+void TestOnlyReloadExperimentsFromConfigVariables();
 #endif
 
 // Print out a list of all experiments that are built into this binary.

test/core/end2end/end2end_test_fuzzer.cc

@@ -124,6 +124,7 @@ DEFINE_PROTO_FUZZER(const core_end2end_test_fuzzer::Msg& msg) {
       grpc_core::OverridesFromFuzzConfigVars(msg.config_vars());
   overrides.default_ssl_roots_file_path = CA_CERT_PATH;
   grpc_core::ConfigVars::SetOverrides(overrides);
+  grpc_core::TestOnlyReloadExperimentsFromConfigVariables();
   grpc_event_engine::experimental::SetEventEngineFactory(
       [actions = msg.event_engine_actions()]() {
         FuzzingEventEngine::Options options;

test/core/end2end/fuzzers/api_fuzzer.cc

@@ -1142,9 +1142,8 @@ DEFINE_PROTO_FUZZER(const api_fuzzer::Msg& msg) {
   if (squelch && !grpc_core::GetEnv("GRPC_TRACE_FUZZER").has_value()) {
     gpr_set_log_function(dont_log);
   }
-  if (msg.has_config_vars()) {
+  grpc_core::ApplyFuzzConfigVars(msg.config_vars());
-    grpc_core::ApplyFuzzConfigVars(msg.config_vars());
+  grpc_core::TestOnlyReloadExperimentsFromConfigVariables();
-  }
   grpc_event_engine::experimental::SetEventEngineFactory(
       [actions = msg.event_engine_actions()]() {
         return std::make_unique<FuzzingEventEngine>(