[resource-quota] Add metrics for number of calls, connections dropped due to quota exceeded (#37833)

Closes #37833

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37833 from ctiller:statty f049916086
PiperOrigin-RevId: 681236201

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -3110,6 +3110,7 @@ static void benign_reclaimer_locked(
   if (error.ok() && t->stream_map.empty()) {
     // Channel with no active streams: send a goaway to try and make it
     // disconnect cleanly
+    grpc_core::global_stats().IncrementRqConnectionsDropped();
     GRPC_TRACE_LOG(resource_quota, INFO)
         << "HTTP2: " << t->peer_string.as_string_view()
         << " - send goaway to free memory";
@@ -3139,6 +3140,7 @@ static void destructive_reclaimer_locked(
     GRPC_TRACE_LOG(resource_quota, INFO)
         << "HTTP2: " << t->peer_string.as_string_view()
         << " - abandon stream id " << s->id;
+    grpc_core::global_stats().IncrementRqCallsDropped();
     grpc_chttp2_cancel_stream(
         t.get(), s,
         grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"),

src/core/telemetry/stats_data.cc

@@ -128,6 +128,8 @@ const absl::string_view
         "client_subchannels_created",
         "server_channels_created",
         "insecure_connections_created",
+        "rq_connections_dropped",
+        "rq_calls_dropped",
         "syscall_write",
         "syscall_read",
         "tcp_read_alloc_8k",
@@ -165,6 +167,8 @@ const absl::string_view GlobalStats::counter_doc[static_cast<int>(
     "Number of client subchannels created",
     "Number of server channels created",
     "Number of insecure connections created",
+    "Number of connections dropped due to resource quota exceeded",
+    "Number of calls dropped due to resource quota exceeded",
     "Number of write syscalls (or equivalent - eg sendmsg) made by this "
     "process",
     "Number of read syscalls (or equivalent - eg recvmsg) made by this process",
@@ -464,6 +468,8 @@ GlobalStats::GlobalStats()
       client_subchannels_created{0},
       server_channels_created{0},
       insecure_connections_created{0},
+      rq_connections_dropped{0},
+      rq_calls_dropped{0},
       syscall_write{0},
       syscall_read{0},
       tcp_read_alloc_8k{0},
@@ -601,6 +607,10 @@ std::unique_ptr<GlobalStats> GlobalStatsCollector::Collect() const {
         data.server_channels_created.load(std::memory_order_relaxed);
     result->insecure_connections_created +=
         data.insecure_connections_created.load(std::memory_order_relaxed);
+    result->rq_connections_dropped +=
+        data.rq_connections_dropped.load(std::memory_order_relaxed);
+    result->rq_calls_dropped +=
+        data.rq_calls_dropped.load(std::memory_order_relaxed);
     result->syscall_write += data.syscall_write.load(std::memory_order_relaxed);
     result->syscall_read += data.syscall_read.load(std::memory_order_relaxed);
     result->tcp_read_alloc_8k +=
@@ -716,6 +726,9 @@ std::unique_ptr<GlobalStats> GlobalStats::Diff(const GlobalStats& other) const {
       server_channels_created - other.server_channels_created;
   result->insecure_connections_created =
       insecure_connections_created - other.insecure_connections_created;
+  result->rq_connections_dropped =
+      rq_connections_dropped - other.rq_connections_dropped;
+  result->rq_calls_dropped = rq_calls_dropped - other.rq_calls_dropped;
   result->syscall_write = syscall_write - other.syscall_write;
   result->syscall_read = syscall_read - other.syscall_read;
   result->tcp_read_alloc_8k = tcp_read_alloc_8k - other.tcp_read_alloc_8k;

src/core/telemetry/stats_data.h

@@ -206,6 +206,8 @@ struct GlobalStats {
     kClientSubchannelsCreated,
     kServerChannelsCreated,
     kInsecureConnectionsCreated,
+    kRqConnectionsDropped,
+    kRqCallsDropped,
     kSyscallWrite,
     kSyscallRead,
     kTcpReadAlloc8k,
@@ -283,6 +285,8 @@ struct GlobalStats {
       uint64_t client_subchannels_created;
       uint64_t server_channels_created;
       uint64_t insecure_connections_created;
+      uint64_t rq_connections_dropped;
+      uint64_t rq_calls_dropped;
       uint64_t syscall_write;
       uint64_t syscall_read;
       uint64_t tcp_read_alloc_8k;
@@ -373,6 +377,13 @@ class GlobalStatsCollector {
     data_.this_cpu().insecure_connections_created.fetch_add(
         1, std::memory_order_relaxed);
   }
+  void IncrementRqConnectionsDropped() {
+    data_.this_cpu().rq_connections_dropped.fetch_add(
+        1, std::memory_order_relaxed);
+  }
+  void IncrementRqCallsDropped() {
+    data_.this_cpu().rq_calls_dropped.fetch_add(1, std::memory_order_relaxed);
+  }
   void IncrementSyscallWrite() {
     data_.this_cpu().syscall_write.fetch_add(1, std::memory_order_relaxed);
   }
@@ -562,6 +573,8 @@ class GlobalStatsCollector {
     std::atomic<uint64_t> client_subchannels_created{0};
     std::atomic<uint64_t> server_channels_created{0};
     std::atomic<uint64_t> insecure_connections_created{0};
+    std::atomic<uint64_t> rq_connections_dropped{0};
+    std::atomic<uint64_t> rq_calls_dropped{0};
     std::atomic<uint64_t> syscall_write{0};
     std::atomic<uint64_t> syscall_read{0};
     std::atomic<uint64_t> tcp_read_alloc_8k{0};

src/core/telemetry/stats_data.yaml

@@ -32,6 +32,11 @@
   doc: Number of server channels created
 - counter: insecure_connections_created
   doc: Number of insecure connections created
+# resource quota
+- counter: rq_connections_dropped
+  doc: Number of connections dropped due to resource quota exceeded
+- counter: rq_calls_dropped
+  doc: Number of calls dropped due to resource quota exceeded
 # tcp
 - counter: syscall_write
   doc: Number of write syscalls (or equivalent - eg sendmsg) made by this process