From 8038d2d87860feecc279346460c7e223fdde7bd9 Mon Sep 17 00:00:00 2001 From: AJ Heller Date: Fri, 24 Mar 2023 17:51:29 -0700 Subject: [PATCH] [EventEngine][Windows] Add shutdown grace period (#32713) This adds a small wait for all timers to finish executing on WindowsEventEngine shutdown. I have seen this wait code triggered on rare occasions where either 10's of tests are running concurrently, or in an environment where IPv6 is detected but disabled, and connection cancellation takes seconds to complete. I have not gotten to the bottom of the latter problem, but feel WindowsEventEngine destruction can tolerate some small extra time. --- .../event_engine/windows/windows_engine.cc | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/core/lib/event_engine/windows/windows_engine.cc b/src/core/lib/event_engine/windows/windows_engine.cc index 1b6fce5a71a..5cfc2e75893 100644 --- a/src/core/lib/event_engine/windows/windows_engine.cc +++ b/src/core/lib/event_engine/windows/windows_engine.cc @@ -109,15 +109,31 @@ WindowsEventEngine::WindowsEventEngine() WindowsEventEngine::~WindowsEventEngine() { GRPC_EVENT_ENGINE_TRACE("~WindowsEventEngine::%p", this); { - grpc_core::MutexLock lock(&task_mu_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) { - for (auto handle : known_handles_) { - gpr_log(GPR_ERROR, - "WindowsEventEngine:%p uncleared TaskHandle at shutdown:%s", - this, HandleToString(handle).c_str()); + task_mu_.Lock(); + if (!known_handles_.empty()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) { + for (auto handle : known_handles_) { + gpr_log(GPR_ERROR, + "WindowsEventEngine:%p uncleared TaskHandle at shutdown:%s", + this, + HandleToString(handle).c_str()); + } + } + // Allow a small grace period for timers to be run before shutting down. + auto deadline = + timer_manager_.Now() + grpc_core::Duration::FromSecondsAsDouble(10); + while (!known_handles_.empty() && timer_manager_.Now() < deadline) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_event_engine_trace)) { + GRPC_LOG_EVERY_N_SEC(1, GPR_DEBUG, "Waiting for timers. %d remaining", + known_handles_.size()); + } + task_mu_.Unlock(); + absl::SleepFor(absl::Milliseconds(200)); + task_mu_.Lock(); } } GPR_ASSERT(GPR_LIKELY(known_handles_.empty())); + task_mu_.Unlock(); } iocp_.Kick(); iocp_worker_.WaitForShutdown();