[xds] Support for multiple servers in the bootstrap.json (#36021)

Closes #36021

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36021 from eugeneo:tasks/multiple-servers-bootstrap 8ebf7e1c06
PiperOrigin-RevId: 615150672

src/core/ext/xds/xds_bootstrap.h

@@ -65,16 +65,14 @@ class XdsBootstrap {
    public:
     virtual ~Authority() = default;
 
-    virtual const XdsServer* server() const = 0;
+    virtual std::vector<const XdsServer*> servers() const = 0;
   };
 
   virtual ~XdsBootstrap() = default;
 
   virtual std::string ToString() const = 0;
 
-  // TODO(roth): We currently support only one server. Fix this when we
-  // add support for fallback for the xds channel.
-  virtual const XdsServer& server() const = 0;
+  virtual std::vector<const XdsServer*> servers() const = 0;
 
   // Returns the node information, or null if not present in the bootstrap
   // config.

src/core/ext/xds/xds_bootstrap_grpc.cc

@@ -14,17 +14,25 @@
 // limitations under the License.
 //
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/ext/xds/xds_bootstrap_grpc.h"
 
+#include <grpc/support/json.h>
+#include <grpc/support/port_platform.h>
 #include <stdlib.h>
 
 #include <algorithm>
 #include <set>
+#include <string>
 #include <utility>
 #include <vector>
 
+#include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/json/json_object_loader.h"
+#include "src/core/lib/json/json_reader.h"
+#include "src/core/lib/json/json_writer.h"
+#include "src/core/lib/security/credentials/channel_creds_registry.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/match.h"
@@ -34,16 +42,6 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
-#include <grpc/support/json.h>
-
-#include "src/core/lib/config/core_configuration.h"
-#include "src/core/lib/gprpp/ref_counted_ptr.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/json/json_object_loader.h"
-#include "src/core/lib/json/json_reader.h"
-#include "src/core/lib/json/json_writer.h"
-#include "src/core/lib/security/credentials/channel_creds_registry.h"
-
 namespace grpc_core {
 
 //
@@ -330,11 +328,14 @@ std::string GrpcXdsBootstrap::ToString() const {
     parts.push_back(
         absl::StrFormat("    client_listener_resource_name_template=\"%s\",\n",
                         entry.second.client_listener_resource_name_template()));
-    if (entry.second.server() != nullptr) {
-      parts.push_back(absl::StrFormat(
-          "    servers=[\n%s\n],\n",
-          JsonDump(static_cast<const GrpcXdsServer*>(entry.second.server())
-                       ->ToJson())));
+    std::vector<std::string> server_jsons;
+    for (const XdsServer* server : entry.second.servers()) {
+      server_jsons.emplace_back(
+          JsonDump(static_cast<const GrpcXdsServer*>(server)->ToJson()));
+    }
+    if (!server_jsons.empty()) {
+      parts.push_back(absl::StrFormat("    servers=[\n%s\n],\n",
+                                      absl::StrJoin(server_jsons, ",\n")));
     }
     parts.push_back("      },\n");
   }

src/core/ext/xds/xds_bootstrap_grpc.h

@@ -104,8 +104,13 @@ class GrpcXdsBootstrap : public XdsBootstrap {
 
   class GrpcAuthority : public Authority {
    public:
-    const XdsServer* server() const override {
-      return servers_.empty() ? nullptr : &servers_[0];
+    std::vector<const XdsServer*> servers() const override {
+      std::vector<const XdsServer*> servers;
+      servers.reserve(servers_.size());
+      for (const auto& server : servers_) {
+        servers.emplace_back(&server);
+      }
+      return servers;
     }
 
     const std::string& client_listener_resource_name_template() const {
@@ -129,7 +134,15 @@ class GrpcXdsBootstrap : public XdsBootstrap {
 
   std::string ToString() const override;
 
-  const XdsServer& server() const override { return servers_[0]; }
+  std::vector<const XdsServer*> servers() const override {
+    std::vector<const XdsServer*> servers;
+    servers.reserve(servers_.size());
+    for (const auto& server : servers_) {
+      servers.emplace_back(&server);
+    }
+    return servers;
+  }
+
   const Node* node() const override {
     return node_.has_value() ? &*node_ : nullptr;
   }

src/core/ext/xds/xds_client.cc

@@ -1584,9 +1584,10 @@ void XdsClient::WatchResource(const XdsResourceType* type,
                        "\" not present in bootstrap config")));
       return;
     }
-    xds_server = authority->server();
+    xds_server =
+        authority->servers().empty() ? nullptr : authority->servers().front();
   }
-  if (xds_server == nullptr) xds_server = &bootstrap_->server();
+  if (xds_server == nullptr) xds_server = bootstrap_->servers().front();
   {
     MutexLock lock(&mu_);
     MaybeRegisterResourceTypeLocked(type);

test/core/memory_usage/memory_usage_test.cc

@@ -210,7 +210,7 @@ XdsServer StartXdsServerAndConfigureBootstrap(
   // Generate xDS bootstrap and set the env var.
   std::string bootstrap =
       grpc::testing::XdsBootstrapBuilder()
-          .SetDefaultServer(absl::StrCat("localhost:", xds_server_port))
+          .SetServers({absl::StrCat("localhost:", xds_server_port)})
           .SetXdsChannelCredentials("insecure")
           .Build();
   grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", bootstrap);

test/core/xds/xds_bootstrap_test.cc

@@ -55,12 +55,30 @@ namespace grpc_core {
 namespace testing {
 namespace {
 
+MATCHER_P2(EqXdsServer, name, creds_config_type, "equals XdsServer") {
+  auto* server = static_cast<const GrpcXdsBootstrap::GrpcXdsServer*>(arg);
+  if (!::testing::ExplainMatchResult(::testing::Ne(nullptr), server,
+                                     result_listener)) {
+    return false;
+  }
+  bool ok = ::testing::ExplainMatchResult(name, server->server_uri(),
+                                          result_listener);
+  auto creds_config = server->channel_creds_config();
+  if (!::testing::ExplainMatchResult(::testing::Ne(nullptr), creds_config,
+                                     result_listener)) {
+    return false;
+  }
+  ok |= ::testing::ExplainMatchResult(creds_config_type, creds_config->type(),
+                                      result_listener);
+  return ok;
+}
+
 TEST(XdsBootstrapTest, Basic) {
   const char* json_str =
       "{"
       "  \"xds_servers\": ["
       "    {"
-      "      \"server_uri\": \"fake:///lb\","
+      "      \"server_uri\": \"fake:///lb1\","
       "      \"channel_creds\": ["
       "        {"
       "          \"type\": \"fake\","
@@ -70,14 +88,11 @@ TEST(XdsBootstrapTest, Basic) {
       "      \"ignore\": 0"
       "    },"
       "    {"
-      "      \"server_uri\": \"ignored\","
+      "      \"server_uri\": \"fake:///lb2\","
       "      \"channel_creds\": ["
       "        {"
-      "          \"type\": \"ignored\","
+      "          \"type\": \"fake\","
       "          \"ignore\": 0"
-      "        },"
-      "        {"
-      "          \"type\": \"fake\""
       "        }"
       "      ],"
       "      \"ignore\": 0"
@@ -97,6 +112,15 @@ TEST(XdsBootstrapTest, Basic) {
       "            }"
       "          ],"
       "          \"server_features\": [\"xds_v3\"]"
+      "        },"
+      "        {"
+      "          \"server_uri\": \"fake:///xds_server2\","
+      "          \"channel_creds\": ["
+      "            {"
+      "              \"type\": \"fake\""
+      "            }"
+      "          ],"
+      "          \"server_features\": [\"xds_v3\"]"
       "        }"
       "      ]"
       "    },"
@@ -106,7 +130,7 @@ TEST(XdsBootstrapTest, Basic) {
       "server/%s\","
       "      \"xds_servers\": ["
       "        {"
-      "          \"server_uri\": \"fake:///xds_server2\","
+      "          \"server_uri\": \"fake:///xds_server3\","
       "          \"channel_creds\": ["
       "            {"
       "              \"type\": \"fake\""
@@ -138,11 +162,9 @@ TEST(XdsBootstrapTest, Basic) {
   auto bootstrap_or = GrpcXdsBootstrap::Create(json_str);
   ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status();
   auto bootstrap = std::move(*bootstrap_or);
-  auto* server =
-      &static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(bootstrap->server());
-  EXPECT_EQ(server->server_uri(), "fake:///lb");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "fake");
+  EXPECT_THAT(bootstrap->servers(),
+              ::testing::ElementsAre(EqXdsServer("fake:///lb1", "fake"),
+                                     EqXdsServer("fake:///lb2", "fake")));
   EXPECT_EQ(bootstrap->authorities().size(), 2);
   auto* authority = static_cast<const GrpcXdsBootstrap::GrpcAuthority*>(
       bootstrap->LookupAuthority("xds.example.com"));
@@ -150,24 +172,18 @@ TEST(XdsBootstrapTest, Basic) {
   EXPECT_EQ(authority->client_listener_resource_name_template(),
             "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/"
             "server/%s");
-  server =
-      static_cast<const GrpcXdsBootstrap::GrpcXdsServer*>(authority->server());
-  ASSERT_NE(server, nullptr);
-  EXPECT_EQ(server->server_uri(), "fake:///xds_server");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "fake");
+  EXPECT_THAT(
+      authority->servers(),
+      ::testing::ElementsAre(EqXdsServer("fake:///xds_server", "fake"),
+                             EqXdsServer("fake:///xds_server2", "fake")));
   authority = static_cast<const GrpcXdsBootstrap::GrpcAuthority*>(
       bootstrap->LookupAuthority("xds.example2.com"));
   ASSERT_NE(authority, nullptr);
   EXPECT_EQ(authority->client_listener_resource_name_template(),
             "xdstp://xds.example2.com/envoy.config.listener.v3.Listener/grpc/"
             "server/%s");
-  server =
-      static_cast<const GrpcXdsBootstrap::GrpcXdsServer*>(authority->server());
-  ASSERT_NE(server, nullptr);
-  EXPECT_EQ(server->server_uri(), "fake:///xds_server2");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "fake");
+  EXPECT_THAT(authority->servers(), ::testing::ElementsAre(EqXdsServer(
+                                        "fake:///xds_server3", "fake")));
   ASSERT_NE(bootstrap->node(), nullptr);
   EXPECT_EQ(bootstrap->node()->id(), "foo");
   EXPECT_EQ(bootstrap->node()->cluster(), "bar");
@@ -203,11 +219,8 @@ TEST(XdsBootstrapTest, ValidWithoutNode) {
   auto bootstrap_or = GrpcXdsBootstrap::Create(json_str);
   ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status();
   auto bootstrap = std::move(*bootstrap_or);
-  auto* server =
-      &static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(bootstrap->server());
-  EXPECT_EQ(server->server_uri(), "fake:///lb");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "fake");
+  EXPECT_THAT(bootstrap->servers(),
+              ::testing::ElementsAre(EqXdsServer("fake:///lb", "fake")));
   EXPECT_EQ(bootstrap->node(), nullptr);
 }
 
@@ -224,11 +237,8 @@ TEST(XdsBootstrapTest, InsecureCreds) {
   auto bootstrap_or = GrpcXdsBootstrap::Create(json_str);
   ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status();
   auto bootstrap = std::move(*bootstrap_or);
-  auto* server =
-      &static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(bootstrap->server());
-  EXPECT_EQ(server->server_uri(), "fake:///lb");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "insecure");
+  EXPECT_THAT(bootstrap->servers(),
+              ::testing::ElementsAre(EqXdsServer("fake:///lb", "insecure")));
   EXPECT_EQ(bootstrap->node(), nullptr);
 }
 
@@ -261,11 +271,8 @@ TEST(XdsBootstrapTest, GoogleDefaultCreds) {
   auto bootstrap_or = GrpcXdsBootstrap::Create(json_str);
   ASSERT_TRUE(bootstrap_or.ok()) << bootstrap_or.status();
   auto bootstrap = std::move(*bootstrap_or);
-  auto* server =
-      &static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(bootstrap->server());
-  EXPECT_EQ(server->server_uri(), "fake:///lb");
-  ASSERT_NE(server->channel_creds_config(), nullptr);
-  EXPECT_EQ(server->channel_creds_config()->type(), "google_default");
+  EXPECT_THAT(bootstrap->servers(), ::testing::ElementsAre(EqXdsServer(
+                                        "fake:///lb", "google_default")));
   EXPECT_EQ(bootstrap->node(), nullptr);
 }
 

test/core/xds/xds_client_fuzzer.cc

@@ -236,13 +236,15 @@ class Fuzzer {
   const XdsBootstrap::XdsServer* GetServer(const std::string& authority) {
     const GrpcXdsBootstrap& bootstrap =
         static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap());
-    if (authority.empty()) return &bootstrap.server();
+    if (authority.empty()) return bootstrap.servers().front();
     const auto* authority_entry =
         static_cast<const GrpcXdsBootstrap::GrpcAuthority*>(
             bootstrap.LookupAuthority(authority));
     if (authority_entry == nullptr) return nullptr;
-    if (authority_entry->server() != nullptr) return authority_entry->server();
-    return &bootstrap.server();
+    if (!authority_entry->servers().empty()) {
+      return authority_entry->servers().front();
+    }
+    return bootstrap.servers().front();
   }
 
   void TriggerConnectionFailure(const std::string& authority,

test/core/xds/xds_client_test.cc

@@ -20,6 +20,12 @@
 
 #include "src/core/ext/xds/xds_client.h"
 
+#include <google/protobuf/any.pb.h>
+#include <google/protobuf/struct.pb.h>
+#include <grpc/grpc.h>
+#include <grpc/support/json.h>
+#include <grpc/support/log.h>
+#include <grpcpp/impl/codegen/config_protobuf.h>
 #include <stdint.h>
 
 #include <algorithm>
@@ -27,23 +33,10 @@
 #include <map>
 #include <memory>
 #include <string>
+#include <vector>
 
-#include <google/protobuf/any.pb.h>
-#include <google/protobuf/struct.pb.h>
-
-#include "absl/strings/str_cat.h"
-#include "absl/time/time.h"
-#include "absl/types/optional.h"
-#include "absl/types/variant.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
-#include "upb/reflection/def.h"
-
-#include <grpc/grpc.h>
-#include <grpc/support/json.h>
-#include <grpc/support/log.h>
-#include <grpcpp/impl/codegen/config_protobuf.h>
-
 #include "src/core/ext/xds/xds_bootstrap.h"
 #include "src/core/ext/xds/xds_resource_type_impl.h"
 #include "src/core/lib/event_engine/default_event_engine.h"
@@ -59,6 +52,11 @@
 #include "test/core/util/scoped_env_var.h"
 #include "test/core/util/test_config.h"
 #include "test/core/xds/xds_transport_fake.h"
+#include "absl/strings/str_cat.h"
+#include "absl/time/time.h"
+#include "absl/types/optional.h"
+#include "absl/types/variant.h"
+#include "upb/reflection/def.h"
 
 // IWYU pragma: no_include <google/protobuf/message.h>
 // IWYU pragma: no_include <google/protobuf/stubs/status.h>
@@ -149,8 +147,12 @@ class XdsClientTest : public ::testing::Test {
 
     class FakeAuthority : public Authority {
      public:
-      const XdsServer* server() const override {
-        return server_.has_value() ? &*server_ : nullptr;
+      std::vector<const XdsServer*> servers() const override {
+        if (server_.has_value()) {
+          return {&*server_};
+        } else {
+          return {};
+        };
       }
 
       void set_server(absl::optional<FakeXdsServer> server) {
@@ -194,7 +196,10 @@ class XdsClientTest : public ::testing::Test {
 
     std::string ToString() const override { return "<fake>"; }
 
-    const XdsServer& server() const override { return server_; }
+    std::vector<const XdsServer*> servers() const override {
+      return {&server_};
+    }
+
     const Node* node() const override {
       return node_.has_value() ? &*node_ : nullptr;
     }
@@ -659,7 +664,8 @@ class XdsClientTest : public ::testing::Test {
 
   RefCountedPtr<FakeXdsTransportFactory::FakeStreamingCall> WaitForAdsStream(
       absl::Duration timeout = absl::Seconds(5)) {
-    return WaitForAdsStream(xds_client_->bootstrap().server(), timeout);
+    return WaitForAdsStream(*xds_client_->bootstrap().servers().front(),
+                            timeout);
   }
 
   // Gets the latest request sent to the fake xDS server.
@@ -1809,7 +1815,7 @@ TEST_F(XdsClientTest, ConnectionFails) {
                /*resource_names=*/{"foo1"});
   CheckRequestNode(*request);  // Should be present on the first request.
   // Transport reports connection failure.
-  TriggerConnectionFailure(xds_client_->bootstrap().server(),
+  TriggerConnectionFailure(*xds_client_->bootstrap().servers().front(),
                            absl::UnavailableError("connection failed"));
   // XdsClient should report an error to the watcher.
   auto error = watcher->WaitForNextError();
@@ -2364,7 +2370,7 @@ TEST_F(XdsClientTest, Federation) {
   // Watcher should initially not see any resource reported.
   EXPECT_FALSE(watcher->HasEvent());
   // XdsClient should have created an ADS stream to the top-level xDS server.
-  auto stream = WaitForAdsStream(xds_client_->bootstrap().server());
+  auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front());
   ASSERT_TRUE(stream != nullptr);
   // XdsClient should have sent a subscription request on the ADS stream.
   auto request = WaitForRequest(stream.get());
@@ -2450,7 +2456,7 @@ TEST_F(XdsClientTest, FederationAuthorityDefaultsToTopLevelXdsServer) {
   // Watcher should initially not see any resource reported.
   EXPECT_FALSE(watcher->HasEvent());
   // XdsClient should have created an ADS stream to the top-level xDS server.
-  auto stream = WaitForAdsStream(xds_client_->bootstrap().server());
+  auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front());
   ASSERT_TRUE(stream != nullptr);
   // XdsClient should have sent a subscription request on the ADS stream.
   auto request = WaitForRequest(stream.get());
@@ -2620,7 +2626,7 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) {
   // Watcher should initially not see any resource reported.
   EXPECT_FALSE(watcher->HasEvent());
   // XdsClient should have created an ADS stream to the top-level xDS server.
-  auto stream = WaitForAdsStream(xds_client_->bootstrap().server());
+  auto stream = WaitForAdsStream(*xds_client_->bootstrap().servers().front());
   ASSERT_TRUE(stream != nullptr);
   // XdsClient should have sent a subscription request on the ADS stream.
   auto request = WaitForRequest(stream.get());

test/core/xds/xds_cluster_resource_type_test.cc

@@ -85,7 +85,8 @@ class XdsClusterTest : public ::testing::Test {
  protected:
   XdsClusterTest()
       : xds_client_(MakeXdsClient()),
-        decode_context_{xds_client_.get(), xds_client_->bootstrap().server(),
+        decode_context_{xds_client_.get(),
+                        *xds_client_->bootstrap().servers().front(),
                         &xds_cluster_resource_type_test_trace,
                         upb_def_pool_.ptr(), upb_arena_.ptr()} {}
 
@@ -1106,7 +1107,7 @@ TEST_F(LrsTest, Valid) {
       static_cast<const XdsClusterResource&>(**decode_result.resource);
   ASSERT_TRUE(resource.lrs_load_reporting_server.has_value());
   EXPECT_EQ(*resource.lrs_load_reporting_server,
-            xds_client_->bootstrap().server());
+            *xds_client_->bootstrap().servers().front());
 }
 
 TEST_F(LrsTest, NotSelfConfigSource) {

test/core/xds/xds_common_types_test.cc

@@ -72,7 +72,8 @@ class XdsCommonTypesTest : public ::testing::Test {
  protected:
   XdsCommonTypesTest()
       : xds_client_(MakeXdsClient()),
-        decode_context_{xds_client_.get(), xds_client_->bootstrap().server(),
+        decode_context_{xds_client_.get(),
+                        *xds_client_->bootstrap().servers().front(),
                         &xds_common_types_test_trace, upb_def_pool_.ptr(),
                         upb_arena_.ptr()} {}
 

test/core/xds/xds_endpoint_resource_type_test.cc

@@ -71,7 +71,8 @@ class XdsEndpointTest : public ::testing::Test {
  protected:
   XdsEndpointTest()
       : xds_client_(MakeXdsClient()),
-        decode_context_{xds_client_.get(), xds_client_->bootstrap().server(),
+        decode_context_{xds_client_.get(),
+                        *xds_client_->bootstrap().servers().front(),
                         &xds_endpoint_resource_type_test_trace,
                         upb_def_pool_.ptr(), upb_arena_.ptr()} {}
 

test/core/xds/xds_listener_resource_type_test.cc

@@ -85,7 +85,8 @@ class XdsListenerTest : public ::testing::Test {
  protected:
   XdsListenerTest()
       : xds_client_(MakeXdsClient()),
-        decode_context_{xds_client_.get(), xds_client_->bootstrap().server(),
+        decode_context_{xds_client_.get(),
+                        *xds_client_->bootstrap().servers().front(),
                         &xds_listener_resource_type_test_trace,
                         upb_def_pool_.ptr(), upb_arena_.ptr()} {}
 

test/core/xds/xds_route_config_resource_type_test.cc

@@ -83,7 +83,8 @@ class XdsRouteConfigTest : public ::testing::Test {
  protected:
   XdsRouteConfigTest()
       : xds_client_(MakeXdsClient()),
-        decode_context_{xds_client_.get(), xds_client_->bootstrap().server(),
+        decode_context_{xds_client_.get(),
+                        *xds_client_->bootstrap().servers().front(),
                         &xds_route_config_resource_type_test_trace,
                         upb_def_pool_.ptr(), upb_arena_.ptr()} {}
 

test/cpp/end2end/xds/xds_cluster_end2end_test.cc

@@ -411,7 +411,7 @@ TEST_P(CdsDeletionTest, ClusterDeleted) {
 
 // Tests that we ignore Cluster deletions if configured to do so.
 TEST_P(CdsDeletionTest, ClusterDeletionIgnored) {
-  InitClient(XdsBootstrapBuilder().SetIgnoreResourceDeletion());
+  InitClient(MakeBootstrapBuilder().SetIgnoreResourceDeletion());
   CreateAndStartBackends(2);
   // Bring up client pointing to backend 0 and wait for it to connect.
   EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});

test/cpp/end2end/xds/xds_core_end2end_test.cc

@@ -324,7 +324,7 @@ TEST_P(GlobalXdsClientTest, InvalidListenerStillExistsIfPreviouslyCached) {
 class TimeoutTest : public XdsEnd2endTest {
  protected:
   void SetUp() override {
-    InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"",
+    InitClient(MakeBootstrapBuilder(), /*lb_expected_authority=*/"",
                /*xds_resource_does_not_exist_timeout_ms=*/2000);
   }
 };
@@ -644,7 +644,7 @@ TEST_P(XdsFederationTest, FederationTargetNoAuthorityWithResourceTemplate) {
   const char* kNewClusterName =
       "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/"
       "new_cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.SetClientDefaultListenerResourceNameTemplate(kNewListenerTemplate);
   builder.AddAuthority(
       kAuthority, absl::StrCat("localhost:", authority_balancer_->port()),
@@ -698,7 +698,7 @@ TEST_P(XdsFederationTest, FederationTargetAuthorityDefaultResourceTemplate) {
   const char* kNewClusterName =
       "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/"
       "cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()));
   InitClient(builder);
@@ -767,7 +767,7 @@ TEST_P(XdsFederationTest, FederationTargetAuthorityWithResourceTemplate) {
   const char* kNewClusterName =
       "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/"
       "cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()),
                        kNewListenerTemplate);
@@ -823,7 +823,7 @@ TEST_P(XdsFederationTest, TargetUriAuthorityUnknown) {
   const char* kNewListenerTemplate =
       "xdstp://xds.example.com/envoy.config.listener.v3.Listener/"
       "client/%s?psm_project_id=1234";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(
       kAuthority, absl::StrCat("localhost:", grpc_pick_unused_port_or_die()),
       kNewListenerTemplate);
@@ -854,7 +854,7 @@ TEST_P(XdsFederationTest, RdsResourceNameAuthorityUnknown) {
   const char* kNewRouteConfigName =
       "xdstp://xds.unknown.com/envoy.config.route.v3.RouteConfiguration/"
       "new_route_config_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()),
                        kNewListenerTemplate);
@@ -900,7 +900,7 @@ TEST_P(XdsFederationTest, CdsResourceNameAuthorityUnknown) {
   const char* kNewClusterName =
       "xdstp://xds.unknown.com/envoy.config.cluster.v3.Cluster/"
       "cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()),
                        kNewListenerTemplate);
@@ -952,7 +952,7 @@ TEST_P(XdsFederationTest, EdsResourceNameAuthorityUnknown) {
   const char* kNewClusterName =
       "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/"
       "cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()),
                        kNewListenerTemplate);
@@ -1019,7 +1019,7 @@ TEST_P(XdsFederationTest, FederationServer) {
   const char* kNewClusterName =
       "xdstp://xds.example.com/envoy.config.cluster.v3.Cluster/"
       "new_cluster_name";
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.SetClientDefaultListenerResourceNameTemplate(kNewListenerTemplate);
   builder.SetServerListenerResourceNameTemplate(kNewServerListenerTemplate);
   builder.AddAuthority(
@@ -1165,7 +1165,7 @@ TEST_P(XdsFederationLoadReportingTest, FederationMultipleLoadReportingTest) {
       "cluster_name";
   const size_t kNumRpcsToDefaultBalancer = 5;
   const size_t kNumRpcsToAuthorityBalancer = 10;
-  XdsBootstrapBuilder builder;
+  XdsBootstrapBuilder builder = MakeBootstrapBuilder();
   builder.AddAuthority(kAuthority,
                        absl::StrCat("localhost:", authority_balancer_->port()),
                        kNewListenerTemplate);
@@ -1276,11 +1276,11 @@ TEST_P(XdsFederationLoadReportingTest, SameServerInAuthorityAndTopLevel) {
   const char* kNewEdsServiceName =
       "xdstp://xds.example.com/envoy.config.endpoint.v3.ClusterLoadAssignment/"
       "edsservice_name";
-  XdsBootstrapBuilder builder;
   std::string xds_server =
       absl::StrCat("localhost:", authority_balancer_->port());
+  XdsBootstrapBuilder builder;
+  builder.SetServers({xds_server});
   builder.AddAuthority(kAuthority, xds_server);
-  builder.SetDefaultServer(xds_server);
   InitClient(builder);
   CreateAndStartBackends(1);
   authority_balancer_->lrs_service()->set_send_all_clusters(true);
@@ -1349,7 +1349,7 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, SecureNamingTest,
 
 // Tests that secure naming check passes if target name is expected.
 TEST_P(SecureNamingTest, TargetNameIsExpected) {
-  InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"localhost:%d");
+  InitClient(MakeBootstrapBuilder(), /*lb_expected_authority=*/"localhost:%d");
   CreateAndStartBackends(4);
   EdsResourceArgs args({
       {"locality0", CreateEndpointsForBackends()},
@@ -1361,7 +1361,7 @@ TEST_P(SecureNamingTest, TargetNameIsExpected) {
 // Tests that secure naming check fails if target name is unexpected.
 TEST_P(SecureNamingTest, TargetNameIsUnexpected) {
   GTEST_FLAG_SET(death_test_style, "threadsafe");
-  InitClient(XdsBootstrapBuilder(),
+  InitClient(MakeBootstrapBuilder(),
              /*lb_expected_authority=*/"incorrect_server_name");
   CreateAndStartBackends(4);
   EdsResourceArgs args({

test/cpp/end2end/xds/xds_csds_end2end_test.cc

@@ -696,7 +696,7 @@ class CsdsShortAdsTimeoutTest : public ClientStatusDiscoveryServiceTest {
  protected:
   void SetUp() override {
     // Shorten the ADS subscription timeout to speed up the test run.
-    InitClient(XdsBootstrapBuilder(), /*lb_expected_authority=*/"",
+    InitClient(absl::nullopt, /*lb_expected_authority=*/"",
                /*xds_resource_does_not_exist_timeout_ms=*/2000);
   }
 };

test/cpp/end2end/xds/xds_end2end_test.cc

@@ -279,7 +279,7 @@ FakeCertificateProvider::CertDataMapWrapper* g_fake2_cert_data_map = nullptr;
 class XdsSecurityTest : public XdsEnd2endTest {
  protected:
   void SetUp() override {
-    XdsBootstrapBuilder builder;
+    XdsBootstrapBuilder builder = MakeBootstrapBuilder();
     builder.AddCertificateProviderPlugin("fake_plugin1", "fake1");
     builder.AddCertificateProviderPlugin("fake_plugin2", "fake2");
     std::vector<std::string> fields;
@@ -799,7 +799,8 @@ class XdsEnabledServerTest : public XdsEnd2endTest {
  protected:
   void SetUp() override {}  // No-op -- individual tests do this themselves.
 
-  void DoSetUp(XdsBootstrapBuilder builder = XdsBootstrapBuilder()) {
+  void DoSetUp(
+      const absl::optional<XdsBootstrapBuilder>& builder = absl::nullopt) {
     InitClient(builder);
     CreateBackends(1, /*xds_enabled=*/true);
     EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
@@ -814,7 +815,7 @@ TEST_P(XdsEnabledServerTest, Basic) {
 }
 
 TEST_P(XdsEnabledServerTest, ListenerDeletionIgnored) {
-  DoSetUp(XdsBootstrapBuilder().SetIgnoreResourceDeletion());
+  DoSetUp(MakeBootstrapBuilder().SetIgnoreResourceDeletion());
   backends_[0]->Start();
   WaitForBackend(DEBUG_LOCATION, 0);
   // Check that we ACKed.
@@ -908,7 +909,7 @@ TEST_P(XdsEnabledServerTest, ListenerAddressMismatch) {
 class XdsServerSecurityTest : public XdsEnd2endTest {
  protected:
   void SetUp() override {
-    XdsBootstrapBuilder builder;
+    XdsBootstrapBuilder builder = MakeBootstrapBuilder();
     builder.AddCertificateProviderPlugin("fake_plugin1", "fake1");
     builder.AddCertificateProviderPlugin("fake_plugin2", "fake2");
     std::vector<std::string> fields;

test/cpp/end2end/xds/xds_end2end_test_lib.cc

@@ -483,9 +483,12 @@ std::vector<int> XdsEnd2endTest::GetBackendPorts(size_t start_index,
   return backend_ports;
 }
 
-void XdsEnd2endTest::InitClient(XdsBootstrapBuilder builder,
+void XdsEnd2endTest::InitClient(absl::optional<XdsBootstrapBuilder> builder,
                                 std::string lb_expected_authority,
                                 int xds_resource_does_not_exist_timeout_ms) {
+  if (!builder.has_value()) {
+    builder = MakeBootstrapBuilder();
+  }
   if (xds_resource_does_not_exist_timeout_ms > 0) {
     xds_channel_args_to_add_.emplace_back(grpc_channel_arg_integer_create(
         const_cast<char*>(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS),
@@ -503,10 +506,7 @@ void XdsEnd2endTest::InitClient(XdsBootstrapBuilder builder,
   }
   xds_channel_args_.num_args = xds_channel_args_to_add_.size();
   xds_channel_args_.args = xds_channel_args_to_add_.data();
-  // Initialize XdsClient state.
-  builder.SetDefaultServer(absl::StrCat("localhost:", balancer_->port()),
-                           /*ignore_if_set=*/true);
-  bootstrap_ = builder.Build();
+  bootstrap_ = builder->Build();
   if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromEnvVar) {
     grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", bootstrap_.c_str());
   } else if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromFile) {

test/cpp/end2end/xds/xds_end2end_test_lib.h

@@ -566,10 +566,15 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType>,
   // Initializes global state for the client, such as the bootstrap file
   // and channel args for the XdsClient.  Then calls ResetStub().
   // All tests must call this exactly once at the start of the test.
-  void InitClient(XdsBootstrapBuilder builder = XdsBootstrapBuilder(),
+  void InitClient(absl::optional<XdsBootstrapBuilder> builder = absl::nullopt,
                   std::string lb_expected_authority = "",
                   int xds_resource_does_not_exist_timeout_ms = 0);
 
+  XdsBootstrapBuilder MakeBootstrapBuilder() {
+    return XdsBootstrapBuilder().SetServers(
+        {absl::StrCat("localhost:", balancer_->port())});
+  }
+
   // Sets channel_, stub_, stub1_, and stub2_.
   void ResetStub(int failover_timeout_ms = 0, ChannelArguments* args = nullptr);
 

test/cpp/end2end/xds/xds_routing_end2end_test.cc

@@ -115,7 +115,7 @@ TEST_P(LdsDeletionTest, ListenerDeleted) {
 
 // Tests that we ignore Listener deletions if configured to do so.
 TEST_P(LdsDeletionTest, ListenerDeletionIgnored) {
-  InitClient(XdsBootstrapBuilder().SetIgnoreResourceDeletion());
+  InitClient(MakeBootstrapBuilder().SetIgnoreResourceDeletion());
   CreateAndStartBackends(2);
   // Bring up client pointing to backend 0 and wait for it to connect.
   EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});

test/cpp/end2end/xds/xds_utils.cc

@@ -60,7 +60,7 @@ using ::envoy::extensions::filters::network::http_connection_manager::v3::
 
 std::string XdsBootstrapBuilder::Build() {
   std::vector<std::string> fields;
-  fields.push_back(MakeXdsServersText(top_server_));
+  fields.push_back(MakeXdsServersText(servers_));
   if (!client_default_listener_resource_name_template_.empty()) {
     fields.push_back(
         absl::StrCat("  \"client_default_listener_resource_name_template\": \"",
@@ -78,9 +78,8 @@ std::string XdsBootstrapBuilder::Build() {
 }
 
 std::string XdsBootstrapBuilder::MakeXdsServersText(
-    absl::string_view server_uri) {
+    absl::Span<const std::string> server_uris) {
   constexpr char kXdsServerTemplate[] =
-      "      \"xds_servers\": [\n"
       "        {\n"
       "          \"server_uri\": \"<SERVER_URI>\",\n"
       "          \"channel_creds\": [\n"
@@ -89,17 +88,21 @@ std::string XdsBootstrapBuilder::MakeXdsServersText(
       "            }\n"
       "          ],\n"
       "          \"server_features\": [<SERVER_FEATURES>]\n"
-      "        }\n"
-      "      ]";
+      "        }";
   std::vector<std::string> server_features;
   if (ignore_resource_deletion_) {
     server_features.push_back("\"ignore_resource_deletion\"");
   }
-  return absl::StrReplaceAll(
-      kXdsServerTemplate,
-      {{"<SERVER_URI>", server_uri},
-       {"<SERVER_CREDS_TYPE>", xds_channel_creds_type_},
-       {"<SERVER_FEATURES>", absl::StrJoin(server_features, ", ")}});
+  std::vector<std::string> servers;
+  for (absl::string_view server_uri : server_uris) {
+    servers.emplace_back(absl::StrReplaceAll(
+        kXdsServerTemplate,
+        {{"<SERVER_URI>", server_uri},
+         {"<SERVER_CREDS_TYPE>", xds_channel_creds_type_},
+         {"<SERVER_FEATURES>", absl::StrJoin(server_features, ", ")}}));
+  }
+  return absl::StrCat("      \"xds_servers\": [\n",
+                      absl::StrJoin(servers, ",\n"), "\n      ]");
 }
 
 std::string XdsBootstrapBuilder::MakeNodeText() {
@@ -148,7 +151,7 @@ std::string XdsBootstrapBuilder::MakeAuthorityText() {
     const std::string& name = p.first;
     const AuthorityInfo& authority_info = p.second;
     std::vector<std::string> fields = {
-        MakeXdsServersText(authority_info.server)};
+        MakeXdsServersText({authority_info.server})};
     if (!authority_info.client_listener_resource_name_template.empty()) {
       fields.push_back(absl::StrCat(
           "\"client_listener_resource_name_template\": \"",

test/cpp/end2end/xds/xds_utils.h

@@ -39,11 +39,8 @@ class XdsBootstrapBuilder {
     ignore_resource_deletion_ = true;
     return *this;
   }
-  // If ignore_if_set is true, sets the default server only if it has
-  // not already been set.
-  XdsBootstrapBuilder& SetDefaultServer(const std::string& server,
-                                        bool ignore_if_set = false) {
-    if (!ignore_if_set || top_server_.empty()) top_server_ = server;
+  XdsBootstrapBuilder& SetServers(absl::Span<const absl::string_view> servers) {
+    servers_ = std::vector<std::string>(servers.begin(), servers.end());
     return *this;
   }
   XdsBootstrapBuilder& SetXdsChannelCredentials(const std::string& type) {
@@ -87,13 +84,13 @@ class XdsBootstrapBuilder {
     std::string client_listener_resource_name_template;
   };
 
-  std::string MakeXdsServersText(absl::string_view server_uri);
+  std::string MakeXdsServersText(absl::Span<const std::string> server_uris);
   std::string MakeNodeText();
   std::string MakeCertificateProviderText();
   std::string MakeAuthorityText();
 
   bool ignore_resource_deletion_ = false;
-  std::string top_server_;
+  std::vector<std::string> servers_;
   std::string xds_channel_creds_type_ = "fake";
   std::string client_default_listener_resource_name_template_;
   std::map<std::string /*key*/, PluginInfo> plugins_;