Merge pull request #16714 from muxi/add-timer-manager-debug

Core infrastructure for timer manager debug

src/core/lib/gpr/sync_posix.cc

@@ -27,6 +27,24 @@
 #include <time.h>
 #include "src/core/lib/profiling/timers.h"
 
+// For debug of the timer manager crash only.
+// TODO (mxyan): remove after bug is fixed.
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+void (*g_grpc_debug_timer_manager_stats)(
+    int64_t timer_manager_init_count, int64_t timer_manager_shutdown_count,
+    int64_t fork_count, int64_t timer_wait_err, int64_t timer_cv_value,
+    int64_t timer_mu_value, int64_t abstime_sec_value,
+    int64_t abstime_nsec_value) = nullptr;
+int64_t g_timer_manager_init_count = 0;
+int64_t g_timer_manager_shutdown_count = 0;
+int64_t g_fork_count = 0;
+int64_t g_timer_wait_err = 0;
+int64_t g_timer_cv_value = 0;
+int64_t g_timer_mu_value = 0;
+int64_t g_abstime_sec_value = -1;
+int64_t g_abstime_nsec_value = -1;
+#endif  // GRPC_DEBUG_TIMER_MANAGER
+
 #ifdef GPR_LOW_LEVEL_COUNTERS
 gpr_atm gpr_mu_locks = 0;
 gpr_atm gpr_counter_atm_cas = 0;
@@ -87,7 +105,31 @@ int gpr_cv_wait(gpr_cv* cv, gpr_mu* mu, gpr_timespec abs_deadline) {
     abs_deadline_ts.tv_sec = static_cast<time_t>(abs_deadline.tv_sec);
     abs_deadline_ts.tv_nsec = abs_deadline.tv_nsec;
     err = pthread_cond_timedwait(cv, mu, &abs_deadline_ts);
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+    // For debug of the timer manager crash only.
+    // TODO (mxyan): remove after bug is fixed.
+    if (GPR_UNLIKELY(!(err == 0 || err == ETIMEDOUT || err == EAGAIN))) {
+      g_abstime_sec_value = abs_deadline_ts.tv_sec;
+      g_abstime_nsec_value = abs_deadline_ts.tv_nsec;
+    }
+#endif
   }
+
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+  // For debug of the timer manager crash only.
+  // TODO (mxyan): remove after bug is fixed.
+  if (GPR_UNLIKELY(!(err == 0 || err == ETIMEDOUT || err == EAGAIN))) {
+    if (g_grpc_debug_timer_manager_stats) {
+      g_timer_wait_err = err;
+      g_timer_cv_value = (int64_t)cv;
+      g_timer_mu_value = (int64_t)mu;
+      g_grpc_debug_timer_manager_stats(
+          g_timer_manager_init_count, g_timer_manager_shutdown_count,
+          g_fork_count, g_timer_wait_err, g_timer_cv_value, g_timer_mu_value,
+          g_abstime_sec_value, g_abstime_nsec_value);
+    }
+  }
+#endif
   GPR_ASSERT(err == 0 || err == ETIMEDOUT || err == EAGAIN);
   return err == ETIMEDOUT;
 }

src/core/lib/iomgr/timer_manager.cc

@@ -61,6 +61,14 @@ static uint64_t g_timed_waiter_generation;
 
 static void timer_thread(void* completed_thread_ptr);
 
+// For debug of the timer manager crash only.
+// TODO (mxyan): remove after bug is fixed.
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+extern int64_t g_timer_manager_init_count;
+extern int64_t g_timer_manager_shutdown_count;
+extern int64_t g_fork_count;
+#endif  // GRPC_DEBUG_TIMER_MANAGER
+
 static void gc_completed_threads(void) {
   if (g_completed_threads != nullptr) {
     completed_thread* to_gc = g_completed_threads;
@@ -284,6 +292,11 @@ static void start_threads(void) {
 void grpc_timer_manager_init(void) {
   gpr_mu_init(&g_mu);
   gpr_cv_init(&g_cv_wait);
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+  // For debug of the timer manager crash only.
+  // TODO (mxyan): remove after bug is fixed.
+  g_timer_manager_init_count++;
+#endif
   gpr_cv_init(&g_cv_shutdown);
   g_threaded = false;
   g_thread_count = 0;
@@ -319,6 +332,11 @@ static void stop_threads(void) {
 }
 
 void grpc_timer_manager_shutdown(void) {
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+  // For debug of the timer manager crash only.
+  // TODO (mxyan): remove after bug is fixed.
+  g_timer_manager_shutdown_count++;
+#endif
   stop_threads();
 
   gpr_mu_destroy(&g_mu);
@@ -327,6 +345,11 @@ void grpc_timer_manager_shutdown(void) {
 }
 
 void grpc_timer_manager_set_threading(bool threaded) {
+#ifdef GRPC_DEBUG_TIMER_MANAGER
+  // For debug of the timer manager crash only.
+  // TODO (mxyan): remove after bug is fixed.
+  g_fork_count++;
+#endif
   if (threaded) {
     start_threads();
   } else {