From adc7affc2d84c5c932158e903596415d3e1a6742 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 1 Oct 2024 18:14:17 -0700 Subject: [PATCH] [resource-quota] Add metrics for number of calls, connections dropped due to quota exceeded (#37833) Closes #37833 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37833 from ctiller:statty f0499160867d52d4d18b679497749ae7c2a6cd86 PiperOrigin-RevId: 681236201 --- .../transport/chttp2/transport/chttp2_transport.cc | 2 ++ src/core/telemetry/stats_data.cc | 13 +++++++++++++ src/core/telemetry/stats_data.h | 13 +++++++++++++ src/core/telemetry/stats_data.yaml | 5 +++++ 4 files changed, 33 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index e4b0b1fdf25..4087efcf6dd 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -3110,6 +3110,7 @@ static void benign_reclaimer_locked( if (error.ok() && t->stream_map.empty()) { // Channel with no active streams: send a goaway to try and make it // disconnect cleanly + grpc_core::global_stats().IncrementRqConnectionsDropped(); GRPC_TRACE_LOG(resource_quota, INFO) << "HTTP2: " << t->peer_string.as_string_view() << " - send goaway to free memory"; @@ -3139,6 +3140,7 @@ static void destructive_reclaimer_locked( GRPC_TRACE_LOG(resource_quota, INFO) << "HTTP2: " << t->peer_string.as_string_view() << " - abandon stream id " << s->id; + grpc_core::global_stats().IncrementRqCallsDropped(); grpc_chttp2_cancel_stream( t.get(), s, grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"), diff --git a/src/core/telemetry/stats_data.cc b/src/core/telemetry/stats_data.cc index b39702f4762..a357b674844 100644 --- a/src/core/telemetry/stats_data.cc +++ b/src/core/telemetry/stats_data.cc @@ -128,6 +128,8 @@ const absl::string_view "client_subchannels_created", "server_channels_created", "insecure_connections_created", + "rq_connections_dropped", + "rq_calls_dropped", "syscall_write", "syscall_read", "tcp_read_alloc_8k", @@ -165,6 +167,8 @@ const absl::string_view GlobalStats::counter_doc[static_cast( "Number of client subchannels created", "Number of server channels created", "Number of insecure connections created", + "Number of connections dropped due to resource quota exceeded", + "Number of calls dropped due to resource quota exceeded", "Number of write syscalls (or equivalent - eg sendmsg) made by this " "process", "Number of read syscalls (or equivalent - eg recvmsg) made by this process", @@ -464,6 +468,8 @@ GlobalStats::GlobalStats() client_subchannels_created{0}, server_channels_created{0}, insecure_connections_created{0}, + rq_connections_dropped{0}, + rq_calls_dropped{0}, syscall_write{0}, syscall_read{0}, tcp_read_alloc_8k{0}, @@ -601,6 +607,10 @@ std::unique_ptr GlobalStatsCollector::Collect() const { data.server_channels_created.load(std::memory_order_relaxed); result->insecure_connections_created += data.insecure_connections_created.load(std::memory_order_relaxed); + result->rq_connections_dropped += + data.rq_connections_dropped.load(std::memory_order_relaxed); + result->rq_calls_dropped += + data.rq_calls_dropped.load(std::memory_order_relaxed); result->syscall_write += data.syscall_write.load(std::memory_order_relaxed); result->syscall_read += data.syscall_read.load(std::memory_order_relaxed); result->tcp_read_alloc_8k += @@ -716,6 +726,9 @@ std::unique_ptr GlobalStats::Diff(const GlobalStats& other) const { server_channels_created - other.server_channels_created; result->insecure_connections_created = insecure_connections_created - other.insecure_connections_created; + result->rq_connections_dropped = + rq_connections_dropped - other.rq_connections_dropped; + result->rq_calls_dropped = rq_calls_dropped - other.rq_calls_dropped; result->syscall_write = syscall_write - other.syscall_write; result->syscall_read = syscall_read - other.syscall_read; result->tcp_read_alloc_8k = tcp_read_alloc_8k - other.tcp_read_alloc_8k; diff --git a/src/core/telemetry/stats_data.h b/src/core/telemetry/stats_data.h index d0e7e5fd267..1ac7579a994 100644 --- a/src/core/telemetry/stats_data.h +++ b/src/core/telemetry/stats_data.h @@ -206,6 +206,8 @@ struct GlobalStats { kClientSubchannelsCreated, kServerChannelsCreated, kInsecureConnectionsCreated, + kRqConnectionsDropped, + kRqCallsDropped, kSyscallWrite, kSyscallRead, kTcpReadAlloc8k, @@ -283,6 +285,8 @@ struct GlobalStats { uint64_t client_subchannels_created; uint64_t server_channels_created; uint64_t insecure_connections_created; + uint64_t rq_connections_dropped; + uint64_t rq_calls_dropped; uint64_t syscall_write; uint64_t syscall_read; uint64_t tcp_read_alloc_8k; @@ -373,6 +377,13 @@ class GlobalStatsCollector { data_.this_cpu().insecure_connections_created.fetch_add( 1, std::memory_order_relaxed); } + void IncrementRqConnectionsDropped() { + data_.this_cpu().rq_connections_dropped.fetch_add( + 1, std::memory_order_relaxed); + } + void IncrementRqCallsDropped() { + data_.this_cpu().rq_calls_dropped.fetch_add(1, std::memory_order_relaxed); + } void IncrementSyscallWrite() { data_.this_cpu().syscall_write.fetch_add(1, std::memory_order_relaxed); } @@ -562,6 +573,8 @@ class GlobalStatsCollector { std::atomic client_subchannels_created{0}; std::atomic server_channels_created{0}; std::atomic insecure_connections_created{0}; + std::atomic rq_connections_dropped{0}; + std::atomic rq_calls_dropped{0}; std::atomic syscall_write{0}; std::atomic syscall_read{0}; std::atomic tcp_read_alloc_8k{0}; diff --git a/src/core/telemetry/stats_data.yaml b/src/core/telemetry/stats_data.yaml index 6b1f04878d0..c9580cded0b 100644 --- a/src/core/telemetry/stats_data.yaml +++ b/src/core/telemetry/stats_data.yaml @@ -32,6 +32,11 @@ doc: Number of server channels created - counter: insecure_connections_created doc: Number of insecure connections created +# resource quota +- counter: rq_connections_dropped + doc: Number of connections dropped due to resource quota exceeded +- counter: rq_calls_dropped + doc: Number of calls dropped due to resource quota exceeded # tcp - counter: syscall_write doc: Number of write syscalls (or equivalent - eg sendmsg) made by this process