diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c
index 3519930f38d..10266c80885 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/security/server_secure_chttp2.c
@@ -124,16 +124,20 @@ static void start(grpc_server *server, void *statep, grpc_pollset **pollsets,
   grpc_tcp_server_start(state->tcp, pollsets, pollset_count, on_accept, state);
 }
 
+static void destroy_done(void *statep) {
+  grpc_server_secure_state *state = statep;
+  grpc_server_listener_destroy_done(state->server);
+  state_unref(state);
+}
+
 /* Server callback: destroy the tcp listener (so we don't generate further
    callbacks) */
 static void destroy(grpc_server *server, void *statep) {
   grpc_server_secure_state *state = statep;
   gpr_mu_lock(&state->mu);
   state->is_shutdown = 1;
-  grpc_tcp_server_destroy(state->tcp, grpc_server_listener_destroy_done,
-                          server);
+  grpc_tcp_server_destroy(state->tcp, destroy_done, state);
   gpr_mu_unlock(&state->mu);
-  state_unref(state);
 }
 
 int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,