[experiments] Optimize `IsExperimentEnabled` (#37057)

Brings measured CPU for this function from 1.95ns down to 0.4ns.

Closes #37057

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37057 from ctiller:xp 99a423016e
PiperOrigin-RevId: 647389088

src/core/lib/experiments/config.cc

@@ -170,6 +170,7 @@ Experiments& ExperimentsSingleton() {
 }  // namespace
 
 void TestOnlyReloadExperimentsFromConfigVariables() {
+  ExperimentFlags::TestOnlyClear();
   ExperimentsSingleton() = LoadExperimentsFromConfigVariable();
   PrintExperimentsList();
 }
@@ -180,8 +181,35 @@ void LoadTestOnlyExperimentsFromMetadata(
       new TestExperiments(experiment_metadata, num_experiments);
 }
 
-bool IsExperimentEnabled(size_t experiment_id) {
-  return ExperimentsSingleton().enabled[experiment_id];
+std::atomic<uint64_t>
+    ExperimentFlags::experiment_flags_[kNumExperimentFlagsWords];
+
+bool ExperimentFlags::LoadFlagsAndCheck(size_t experiment_id) {
+  static_assert(kNumExperiments < kNumExperimentFlagsWords * kFlagsPerWord,
+                "kNumExperiments must be less than "
+                "kNumExperimentFlagsWords*kFlagsPerWord; if this fails then "
+                "make kNumExperimentFlagsWords bigger.");
+  const auto& experiments = ExperimentsSingleton();
+  uint64_t building[kNumExperimentFlagsWords];
+  for (size_t i = 0; i < kNumExperimentFlagsWords; i++) {
+    building[i] = kLoadedFlag;
+  }
+  for (size_t i = 0; i < kNumExperiments; i++) {
+    if (!experiments.enabled[i]) continue;
+    auto bit = i % kFlagsPerWord;
+    auto word = i / kFlagsPerWord;
+    building[word] |= 1ull << bit;
+  }
+  for (size_t i = 0; i < kNumExperimentFlagsWords; i++) {
+    experiment_flags_[i].store(building[i], std::memory_order_relaxed);
+  }
+  return experiments.enabled[experiment_id];
+}
+
+void ExperimentFlags::TestOnlyClear() {
+  for (size_t i = 0; i < kNumExperimentFlagsWords; i++) {
+    experiment_flags_[i].store(0, std::memory_order_relaxed);
+  }
 }
 
 bool IsExperimentEnabledInConfiguration(size_t experiment_id) {

src/core/lib/experiments/config.h

@@ -18,6 +18,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <atomic>
+
 #include "absl/functional/any_invocable.h"
 #include "absl/strings/string_view.h"
 
@@ -38,16 +40,68 @@ struct ExperimentMetadata {
 };
 
 #ifndef GRPC_EXPERIMENTS_ARE_FINAL
+class ExperimentFlags {
+ public:
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsExperimentEnabled(
+      size_t experiment_id) {
+    auto bit = experiment_id % kFlagsPerWord;
+    auto word = experiment_id / kFlagsPerWord;
+    auto cur = experiment_flags_[word].load(std::memory_order_relaxed);
+    if (cur & (1ull << bit)) return true;
+    if (cur & kLoadedFlag) return false;
+    return LoadFlagsAndCheck(experiment_id);
+  }
+
+  template <size_t kExperimentId>
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static bool IsExperimentEnabled() {
+    auto bit = kExperimentId % kFlagsPerWord;
+    auto word = kExperimentId / kFlagsPerWord;
+    auto cur = experiment_flags_[word].load(std::memory_order_relaxed);
+    if (cur & (1ull << bit)) return true;
+    if (cur & kLoadedFlag) return false;
+    return LoadFlagsAndCheck(kExperimentId);
+  }
+
+  static void TestOnlyClear();
+
+ private:
+  static bool LoadFlagsAndCheck(size_t experiment_id);
+
+  // We layout experiment flags in groups of 63... each 64 bit word contains
+  // 63 enablement flags (one per experiment), and the high bit which indicates
+  // whether the flags have been loaded from the configuration.
+  // Consequently, with one load, we can tell if the experiment is definitely
+  // enabled (the bit is set), or definitely disabled (the bit is clear, and the
+  // loaded flag is set), or if we need to load the flags and re-check.
+
+  static constexpr size_t kNumExperimentFlagsWords = 8;
+  static constexpr size_t kFlagsPerWord = 63;
+  static constexpr uint64_t kLoadedFlag = 0x8000000000000000ull;
+  static std::atomic<uint64_t> experiment_flags_[kNumExperimentFlagsWords];
+};
+
 // Return true if experiment \a experiment_id is enabled.
 // Experiments are numbered by their order in the g_experiment_metadata array
 // declared in experiments.h.
-bool IsExperimentEnabled(size_t experiment_id);
+inline bool IsExperimentEnabled(size_t experiment_id) {
+  return ExperimentFlags::IsExperimentEnabled(experiment_id);
+}
+
+template <size_t kExperimentId>
+inline bool IsExperimentEnabled() {
+  return ExperimentFlags::IsExperimentEnabled<kExperimentId>();
+}
 
 // Given a test experiment id, returns true if the test experiment is enabled.
 // Test experiments can be loaded using the LoadTestOnlyExperimentsFromMetadata
 // method.
 bool IsTestExperimentEnabled(size_t experiment_id);
 
+template <size_t kExperimentId>
+inline bool IsTestExperimentEnabled() {
+  return IsTestExperimentEnabled(kExperimentId);
+}
+
 // Slow check for if a named experiment is enabled.
 // Parses the configuration and looks up the experiment in that, so it does not
 // affect any global state, but it does require parsing the configuration every

src/core/lib/experiments/experiments.h

@@ -193,103 +193,103 @@ enum ExperimentIds {
 };
 #define GRPC_EXPERIMENT_IS_INCLUDED_CALL_STATUS_OVERRIDE_ON_CANCELLATION
 inline bool IsCallStatusOverrideOnCancellationEnabled() {
-  return IsExperimentEnabled(kExperimentIdCallStatusOverrideOnCancellation);
+  return IsExperimentEnabled<kExperimentIdCallStatusOverrideOnCancellation>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_CANARY_CLIENT_PRIVACY
 inline bool IsCanaryClientPrivacyEnabled() {
-  return IsExperimentEnabled(kExperimentIdCanaryClientPrivacy);
+  return IsExperimentEnabled<kExperimentIdCanaryClientPrivacy>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_PRIVACY
 inline bool IsClientPrivacyEnabled() {
-  return IsExperimentEnabled(kExperimentIdClientPrivacy);
+  return IsExperimentEnabled<kExperimentIdClientPrivacy>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_CLIENT
 inline bool IsEventEngineClientEnabled() {
-  return IsExperimentEnabled(kExperimentIdEventEngineClient);
+  return IsExperimentEnabled<kExperimentIdEventEngineClient>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_DNS
 inline bool IsEventEngineDnsEnabled() {
-  return IsExperimentEnabled(kExperimentIdEventEngineDns);
+  return IsExperimentEnabled<kExperimentIdEventEngineDns>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER
 inline bool IsEventEngineListenerEnabled() {
-  return IsExperimentEnabled(kExperimentIdEventEngineListener);
+  return IsExperimentEnabled<kExperimentIdEventEngineListener>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_FREE_LARGE_ALLOCATOR
 inline bool IsFreeLargeAllocatorEnabled() {
-  return IsExperimentEnabled(kExperimentIdFreeLargeAllocator);
+  return IsExperimentEnabled<kExperimentIdFreeLargeAllocator>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_HTTP2_STATS_FIX
 inline bool IsHttp2StatsFixEnabled() {
-  return IsExperimentEnabled(kExperimentIdHttp2StatsFix);
+  return IsExperimentEnabled<kExperimentIdHttp2StatsFix>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_FIX
 inline bool IsKeepaliveFixEnabled() {
-  return IsExperimentEnabled(kExperimentIdKeepaliveFix);
+  return IsExperimentEnabled<kExperimentIdKeepaliveFix>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_SERVER_FIX
 inline bool IsKeepaliveServerFixEnabled() {
-  return IsExperimentEnabled(kExperimentIdKeepaliveServerFix);
+  return IsExperimentEnabled<kExperimentIdKeepaliveServerFix>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_MAX_PINGS_WO_DATA_THROTTLE
 inline bool IsMaxPingsWoDataThrottleEnabled() {
-  return IsExperimentEnabled(kExperimentIdMaxPingsWoDataThrottle);
+  return IsExperimentEnabled<kExperimentIdMaxPingsWoDataThrottle>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT
 inline bool IsMonitoringExperimentEnabled() {
-  return IsExperimentEnabled(kExperimentIdMonitoringExperiment);
+  return IsExperimentEnabled<kExperimentIdMonitoringExperiment>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_MULTIPING
 inline bool IsMultipingEnabled() {
-  return IsExperimentEnabled(kExperimentIdMultiping);
+  return IsExperimentEnabled<kExperimentIdMultiping>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_PEER_STATE_BASED_FRAMING
 inline bool IsPeerStateBasedFramingEnabled() {
-  return IsExperimentEnabled(kExperimentIdPeerStateBasedFraming);
+  return IsExperimentEnabled<kExperimentIdPeerStateBasedFraming>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_PICK_FIRST_NEW
 inline bool IsPickFirstNewEnabled() {
-  return IsExperimentEnabled(kExperimentIdPickFirstNew);
+  return IsExperimentEnabled<kExperimentIdPickFirstNew>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_INPROC_TRANSPORT
 inline bool IsPromiseBasedInprocTransportEnabled() {
-  return IsExperimentEnabled(kExperimentIdPromiseBasedInprocTransport);
+  return IsExperimentEnabled<kExperimentIdPromiseBasedInprocTransport>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_RSTPIT
 inline bool IsRstpitEnabled() {
-  return IsExperimentEnabled(kExperimentIdRstpit);
+  return IsExperimentEnabled<kExperimentIdRstpit>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_SCHEDULE_CANCELLATION_OVER_WRITE
 inline bool IsScheduleCancellationOverWriteEnabled() {
-  return IsExperimentEnabled(kExperimentIdScheduleCancellationOverWrite);
+  return IsExperimentEnabled<kExperimentIdScheduleCancellationOverWrite>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_SERVER_PRIVACY
 inline bool IsServerPrivacyEnabled() {
-  return IsExperimentEnabled(kExperimentIdServerPrivacy);
+  return IsExperimentEnabled<kExperimentIdServerPrivacy>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_FRAME_SIZE_TUNING
 inline bool IsTcpFrameSizeTuningEnabled() {
-  return IsExperimentEnabled(kExperimentIdTcpFrameSizeTuning);
+  return IsExperimentEnabled<kExperimentIdTcpFrameSizeTuning>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_RCV_LOWAT
 inline bool IsTcpRcvLowatEnabled() {
-  return IsExperimentEnabled(kExperimentIdTcpRcvLowat);
+  return IsExperimentEnabled<kExperimentIdTcpRcvLowat>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TRACE_RECORD_CALLOPS
 inline bool IsTraceRecordCallopsEnabled() {
-  return IsExperimentEnabled(kExperimentIdTraceRecordCallops);
+  return IsExperimentEnabled<kExperimentIdTraceRecordCallops>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_UNCONSTRAINED_MAX_QUOTA_BUFFER_SIZE
 inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() {
-  return IsExperimentEnabled(kExperimentIdUnconstrainedMaxQuotaBufferSize);
+  return IsExperimentEnabled<kExperimentIdUnconstrainedMaxQuotaBufferSize>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_CLEARS_TIME_CACHE
 inline bool IsWorkSerializerClearsTimeCacheEnabled() {
-  return IsExperimentEnabled(kExperimentIdWorkSerializerClearsTimeCache);
+  return IsExperimentEnabled<kExperimentIdWorkSerializerClearsTimeCache>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_DISPATCH
 inline bool IsWorkSerializerDispatchEnabled() {
-  return IsExperimentEnabled(kExperimentIdWorkSerializerDispatch);
+  return IsExperimentEnabled<kExperimentIdWorkSerializerDispatch>();
 }
 
 extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];

test/core/experiments/bm_experiments.cc

@@ -18,7 +18,7 @@
 
 static void BM_IsExperimentEnabled(benchmark::State& state) {
   for (auto _ : state) {
-    grpc_core::IsMonitoringExperimentEnabled();
+    benchmark::DoNotOptimize(grpc_core::IsMonitoringExperimentEnabled());
   }
 }
 BENCHMARK(BM_IsExperimentEnabled);

test/core/experiments/fixtures/experiments.h

@@ -133,19 +133,19 @@ enum ExperimentIds {
 };
 #define GRPC_EXPERIMENT_IS_INCLUDED_TEST_EXPERIMENT_1
 inline bool IsTestExperiment1Enabled() {
-  return IsTestExperimentEnabled(kExperimentIdTestExperiment1);
+  return IsTestExperimentEnabled<kExperimentIdTestExperiment1>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TEST_EXPERIMENT_2
 inline bool IsTestExperiment2Enabled() {
-  return IsTestExperimentEnabled(kExperimentIdTestExperiment2);
+  return IsTestExperimentEnabled<kExperimentIdTestExperiment2>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TEST_EXPERIMENT_3
 inline bool IsTestExperiment3Enabled() {
-  return IsTestExperimentEnabled(kExperimentIdTestExperiment3);
+  return IsTestExperimentEnabled<kExperimentIdTestExperiment3>();
 }
 #define GRPC_EXPERIMENT_IS_INCLUDED_TEST_EXPERIMENT_4
 inline bool IsTestExperiment4Enabled() {
-  return IsTestExperimentEnabled(kExperimentIdTestExperiment4);
+  return IsTestExperimentEnabled<kExperimentIdTestExperiment4>();
 }
 
 extern const ExperimentMetadata g_test_experiment_metadata[kNumTestExperiments];

tools/codegen/core/experiments_compiler.py

@@ -456,7 +456,7 @@ class ExperimentsCompiler(object):
                 )
                 print(
                     "inline bool Is%sEnabled() { return"
-                    " Is%sExperimentEnabled(kExperimentId%s); }"
+                    " Is%sExperimentEnabled<kExperimentId%s>(); }"
                     % (
                         SnakeToPascal(exp.name),
                         "Test" if mode == "test" else "",