Fix shutdown path

src/core/lib/iomgr/executor.c

@@ -90,8 +90,7 @@ static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
     grpc_closure *next = c->next_data.next;
     grpc_error *error = c->error_data.error;
 #ifndef NDEBUG
-    GPR_ASSERT(!c->scheduled);
-    c->scheduled = true;
+    c->scheduled = false;
 #endif
     c->cb(exec_ctx, c->cb_arg, error);
     GRPC_ERROR_UNREF(error);
@@ -111,6 +110,7 @@ void grpc_executor_shutdown(grpc_exec_ctx *exec_ctx) {
   for (gpr_atm i = 0; i < g_cur_threads; i++) {
     gpr_thd_join(g_thread_state[i].id);
   }
+  gpr_atm_no_barrier_store(&g_cur_threads, 0);
   for (size_t i = 0; i < g_max_threads; i++) {
     gpr_mu_destroy(&g_thread_state[i].mu);
     gpr_cv_destroy(&g_thread_state[i].cv);
@@ -147,8 +147,12 @@ static void executor_thread(void *arg) {
 
 static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                           grpc_error *error) {
-  thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
   size_t cur_thread_count = (size_t)gpr_atm_no_barrier_load(&g_cur_threads);
+  if (cur_thread_count == 0) {
+    grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
+    return;
+  }
+  thread_state *ts = (thread_state *)gpr_tls_get(&g_this_thread_state);
   if (ts == NULL) {
     ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
   }

src/core/lib/iomgr/iomgr.c

@@ -44,6 +44,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -61,6 +62,7 @@ void grpc_iomgr_init(void) {
   gpr_mu_init(&g_mu);
   gpr_cv_init(&g_rcv);
   grpc_exec_ctx_global_init();
+  grpc_executor_init();
   grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC));
   g_root_object.next = g_root_object.prev = &g_root_object;
   g_root_object.name = "root";
@@ -143,6 +145,8 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) {
 
   grpc_timer_list_shutdown(exec_ctx);
   grpc_exec_ctx_flush(exec_ctx);
+  grpc_executor_shutdown(exec_ctx);
+  grpc_exec_ctx_flush(exec_ctx);
 
   /* ensure all threads have left g_mu */
   gpr_mu_lock(&g_mu);

src/core/lib/surface/init.c

@@ -155,7 +155,6 @@ void grpc_init(void) {
 #endif
     grpc_security_pre_init();
     grpc_iomgr_init();
-    grpc_executor_init();
     gpr_timers_global_init();
     grpc_handshaker_factory_registry_init();
     grpc_security_init();
@@ -180,10 +179,10 @@ void grpc_init(void) {
 void grpc_shutdown(void) {
   int i;
   GRPC_API_TRACE("grpc_shutdown(void)", 0, ());
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_exec_ctx exec_ctx =
+      GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
   gpr_mu_lock(&g_init_mu);
   if (--g_initializations == 0) {
-    grpc_executor_shutdown(&exec_ctx);
     grpc_iomgr_shutdown(&exec_ctx);
     gpr_timers_global_destroy();
     grpc_tracer_shutdown();