Merge pull request #21709 from yashykt/20928fix

Fix Issue 20928 and potentially other related xds issues

src/core/ext/filters/client_channel/lb_policy/xds/xds.cc

@@ -746,8 +746,11 @@ void XdsLb::ShutdownLocked() {
   }
   fallback_policy_.reset();
   pending_fallback_policy_.reset();
-  // Cancel the endpoint watch here instead of in our dtor, because the
-  // watcher holds a ref to us.
+  // Cancel the endpoint watch here instead of in our dtor if we are using the
+  // XdsResolver, because the watcher holds a ref to us and we might not be
+  // destroying the Xds client leading to a situation where the Xds lb policy is
+  // never destroyed.
+  if (xds_client_from_channel_ != nullptr) {
     xds_client()->CancelEndpointDataWatch(StringView(eds_service_name()),
                                           endpoint_watcher_);
     if (config_->lrs_load_reporting_server_name().has_value()) {
@@ -759,6 +762,7 @@ void XdsLb::ShutdownLocked() {
           StringView(eds_service_name()), &client_stats_);
     }
     xds_client_from_channel_.reset();
+  }
   xds_client_.reset();
 }