diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc
index 6999704e138..1de5d46dc9b 100644
--- a/src/core/lib/surface/server.cc
+++ b/src/core/lib/surface/server.cc
@@ -1600,13 +1600,6 @@ void grpc_server_shutdown_and_notify(grpc_server* server,
 
   channel_broadcaster_shutdown(&broadcaster, true /* send_goaway */,
                                GRPC_ERROR_NONE);
-
-  if (server->default_resource_user != nullptr) {
-    grpc_resource_quota_unref(
-        grpc_resource_user_quota(server->default_resource_user));
-    grpc_resource_user_shutdown(server->default_resource_user);
-    grpc_resource_user_unref(server->default_resource_user);
-  }
 }
 
 void grpc_server_cancel_all_calls(grpc_server* server) {
@@ -1637,6 +1630,12 @@ void grpc_server_destroy(grpc_server* server) {
   GPR_ASSERT(server->listeners_destroyed == server->listeners.size());
   gpr_mu_unlock(&server->mu_global);
 
+  if (server->default_resource_user != nullptr) {
+    grpc_resource_quota_unref(
+        grpc_resource_user_quota(server->default_resource_user));
+    grpc_resource_user_shutdown(server->default_resource_user);
+    grpc_resource_user_unref(server->default_resource_user);
+  }
   server_unref(server);
 }