diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc
index 28c5f745009..684773e5017 100644
--- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc
+++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc
@@ -746,20 +746,24 @@ void XdsLb::ShutdownLocked() {
   }
   fallback_policy_.reset();
   pending_fallback_policy_.reset();
-  // Cancel the endpoint watch here instead of in our dtor, because the
-  // watcher holds a ref to us.
-  xds_client()->CancelEndpointDataWatch(StringView(eds_service_name()),
-                                        endpoint_watcher_);
-  if (config_->lrs_load_reporting_server_name().has_value()) {
-    // TODO(roth): We should pass the cluster name (in addition to the
-    // eds_service_name) when adding the client stats. To do so, we need to
-    // first find a way to plumb the cluster name down into this LB policy.
-    xds_client()->RemoveClientStats(
-        StringView(config_->lrs_load_reporting_server_name().value().c_str()),
-        StringView(eds_service_name()), &client_stats_);
-  }
-  xds_client_from_channel_.reset();
-  xds_client_.reset();
+  // Cancel the endpoint watch here instead of in our dtor if we are using the
+  // XdsResolver, because the watcher holds a ref to us and we might not be
+  // destroying the Xds client leading to a situation where the Xds lb policy is
+  // never destroyed.
+  if (xds_client_from_channel_ != nullptr) {
+    xds_client()->CancelEndpointDataWatch(StringView(eds_service_name()),
+                                          endpoint_watcher_);
+    if (config_->lrs_load_reporting_server_name().has_value()) {
+      // TODO(roth): We should pass the cluster name (in addition to the
+      // eds_service_name) when adding the client stats. To do so, we need to
+      // first find a way to plumb the cluster name down into this LB policy.
+      xds_client()->RemoveClientStats(
+          StringView(config_->lrs_load_reporting_server_name().value().c_str()),
+          StringView(eds_service_name()), &client_stats_);
+    }
+    xds_client_from_channel_.reset();
+    xds_client_.reset();
+  }
 }
 
 //