diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc
index e6c81d9d460..810b35de2ff 100644
--- a/src/core/lib/surface/server.cc
+++ b/src/core/lib/surface/server.cc
@@ -1352,13 +1352,6 @@ void grpc_server_shutdown_and_notify(grpc_server* server,
 
   channel_broadcaster_shutdown(&broadcaster, true /* send_goaway */,
                                GRPC_ERROR_NONE);
-
-  if (server->default_resource_user != nullptr) {
-    grpc_resource_quota_unref(
-        grpc_resource_user_quota(server->default_resource_user));
-    grpc_resource_user_shutdown(server->default_resource_user);
-    grpc_resource_user_unref(server->default_resource_user);
-  }
 }
 
 void grpc_server_cancel_all_calls(grpc_server* server) {
@@ -1396,6 +1389,12 @@ void grpc_server_destroy(grpc_server* server) {
 
   gpr_mu_unlock(&server->mu_global);
 
+  if (server->default_resource_user != nullptr) {
+    grpc_resource_quota_unref(
+        grpc_resource_user_quota(server->default_resource_user));
+    grpc_resource_user_shutdown(server->default_resource_user);
+    grpc_resource_user_unref(server->default_resource_user);
+  }
   server_unref(server);
 }