From 1761685fc1f190ea1bafa6c5f64bcc31ebd003e8 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 3 Jul 2024 16:01:26 -0700 Subject: [PATCH] [xDS e2e tests] set channel/server creds in tests instead of as a test parameter (#37156) As part of the xDS authority rewriting implementation in #37087, I need to write some tests showing the authority being rewritten. However, the fake security connector currently crashes on unexpected authorities. (As a side note, I think the fake security connector is very cumbersome and should be redesigned, but that's a separate project.) As a result, I need a way to use InsecureCreds on a per-test basis. I thought about just adding an option to `XdsTestType` to trigger use of InsecureCreds, but the logic we use for determining which creds type to use for what is already very cumbersome, and adding another option there would have just made that worse. Instead, I have switched to a simpler approach where the individual tests can decide what creds type to use directly. This both unblocks my other PR and makes the existing code more maintainable. Closes #37156 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37156 from markdroth:xds_e2e_test_creds_improvement 2e813834d09ccdf858868f7f47100d150e92490b PiperOrigin-RevId: 649225248 --- test/cpp/end2end/xds/xds_core_end2end_test.cc | 9 +- test/cpp/end2end/xds/xds_end2end_test.cc | 84 +++------ test/cpp/end2end/xds/xds_end2end_test_lib.cc | 176 +++++++++--------- test/cpp/end2end/xds/xds_end2end_test_lib.h | 116 ++++++------ 4 files changed, 174 insertions(+), 211 deletions(-) diff --git a/test/cpp/end2end/xds/xds_core_end2end_test.cc b/test/cpp/end2end/xds/xds_core_end2end_test.cc index 6b8fcd43482..f3b6d78f901 100644 --- a/test/cpp/end2end/xds/xds_core_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_core_end2end_test.cc @@ -166,6 +166,9 @@ TEST_P(XdsClientTest, XdsStreamErrorPropagation) { class XdsServerTlsTest : public XdsEnd2endTest { protected: + XdsServerTlsTest() + : XdsEnd2endTest(/*balancer_credentials=*/CreateTlsServerCredentials()) {} + void SetUp() override { InitClient(MakeBootstrapBuilder().SetXdsChannelCredentials( "tls", absl::StrCat("{\"ca_certificate_file\": \"", @@ -176,10 +179,8 @@ class XdsServerTlsTest : public XdsEnd2endTest { } }; -INSTANTIATE_TEST_SUITE_P( - XdsTest, XdsServerTlsTest, - ::testing::Values(XdsTestType().set_xds_server_uses_tls_creds(true)), - &XdsTestType::Name); +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerTlsTest, + ::testing::Values(XdsTestType()), &XdsTestType::Name); TEST_P(XdsServerTlsTest, Basic) { CreateAndStartBackends(1); diff --git a/test/cpp/end2end/xds/xds_end2end_test.cc b/test/cpp/end2end/xds/xds_end2end_test.cc index 10f6e5febf1..e80255d18c1 100644 --- a/test/cpp/end2end/xds/xds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_end2end_test.cc @@ -292,8 +292,12 @@ class XdsSecurityTest : public XdsEnd2endTest { kCaCertPath)); builder.AddCertificateProviderPlugin("file_plugin", "file_watcher", absl::StrJoin(fields, ",\n")); - InitClient(builder); - CreateAndStartBackends(2); + InitClient(builder, /*lb_expected_authority=*/"", + /*xds_resource_does_not_exist_timeout_ms=*/0, + /*balancer_authority_override=*/"", + CreateXdsChannelCredentials()); + CreateAndStartBackends(2, /*xds_enabled=*/false, + CreateMtlsServerCredentials()); root_cert_ = grpc_core::testing::GetFileContents(kCaCertPath); bad_root_cert_ = grpc_core::testing::GetFileContents(kBadClientCertPath); identity_pair_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath); @@ -921,8 +925,12 @@ class XdsServerSecurityTest : public XdsEnd2endTest { kCaCertPath)); builder.AddCertificateProviderPlugin("file_plugin", "file_watcher", absl::StrJoin(fields, ",\n")); - InitClient(builder); - CreateBackends(1, /*xds_enabled=*/true); + InitClient(builder, /*lb_expected_authority=*/"", + /*xds_resource_does_not_exist_timeout_ms=*/0, + /*balancer_authority_override=*/"", + CreateXdsChannelCredentials()); + CreateBackends(1, /*xds_enabled=*/true, + XdsServerCredentials(InsecureServerCredentials())); root_cert_ = grpc_core::testing::GetFileContents(kCaCertPath); bad_root_cert_ = grpc_core::testing::GetFileContents(kBadClientCertPath); identity_pair_ = ReadTlsIdentityPair(kServerKeyPath, kServerCertPath); @@ -3106,14 +3114,8 @@ TEST_P(XdsRbacTestWithActionAndAuditConditionPermutations, MultipleLoggers) { } } -// CDS depends on XdsResolver. -// Security depends on v3. -// Not enabling load reporting or RDS, since those are irrelevant to these -// tests. -INSTANTIATE_TEST_SUITE_P( - XdsTest, XdsSecurityTest, - ::testing::Values(XdsTestType().set_use_xds_credentials()), - &XdsTestType::Name); +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsSecurityTest, + ::testing::Values(XdsTestType()), &XdsTestType::Name); // We are only testing the server here. // Run with bootstrap from env var, so that we use a global XdsClient @@ -3126,39 +3128,28 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerTest, // We are only testing the server here. // Run with bootstrap from env var so that we use one XdsClient. -INSTANTIATE_TEST_SUITE_P( - XdsTest, XdsServerSecurityTest, - ::testing::Values( - XdsTestType() - .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar) - .set_use_xds_credentials()), - &XdsTestType::Name); +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerSecurityTest, + ::testing::Values(XdsTestType().set_bootstrap_source( + XdsTestType::kBootstrapFromEnvVar)), + &XdsTestType::Name); -INSTANTIATE_TEST_SUITE_P( - XdsTest, XdsEnabledServerStatusNotificationTest, - ::testing::Values(XdsTestType().set_use_xds_credentials()), - &XdsTestType::Name); +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerStatusNotificationTest, + ::testing::Values(XdsTestType()), &XdsTestType::Name); // Run with bootstrap from env var so that we use one XdsClient. -INSTANTIATE_TEST_SUITE_P( - XdsTest, XdsServerFilterChainMatchTest, - ::testing::Values( - XdsTestType() - .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar) - .set_use_xds_credentials()), - &XdsTestType::Name); +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerFilterChainMatchTest, + ::testing::Values(XdsTestType().set_bootstrap_source( + XdsTestType::kBootstrapFromEnvVar)), + &XdsTestType::Name); // Test xDS-enabled server with and without RDS. // Run with bootstrap from env var so that we use one XdsClient. INSTANTIATE_TEST_SUITE_P( XdsTest, XdsServerRdsTest, ::testing::Values( + XdsTestType().set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar) - .set_use_xds_credentials(), - XdsTestType() - .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar) - .set_use_xds_credentials() .set_enable_rds_testing()), &XdsTestType::Name); @@ -3169,19 +3160,14 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P( XdsTest, XdsRbacTest, ::testing::Values( - XdsTestType().set_use_xds_credentials().set_bootstrap_source( + XdsTestType().set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), + XdsTestType().set_enable_rds_testing().set_bootstrap_source( XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() - .set_enable_rds_testing() - .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), - XdsTestType() - .set_use_xds_credentials() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) @@ -3196,12 +3182,10 @@ INSTANTIATE_TEST_SUITE_P( XdsTest, XdsRbacTestWithRouteOverrideAlwaysPresent, ::testing::Values( XdsTestType() - .set_use_xds_credentials() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) @@ -3216,44 +3200,36 @@ INSTANTIATE_TEST_SUITE_P( XdsTest, XdsRbacTestWithActionPermutations, ::testing::Values( XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_DENY) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_rbac_action(RBAC_Action_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_rbac_action(RBAC_Action_DENY) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) .set_rbac_action(RBAC_Action_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) .set_rbac_action(RBAC_Action_DENY) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) .set_rbac_action(RBAC_Action_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_filter_config_setup( XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute) @@ -3265,37 +3241,31 @@ INSTANTIATE_TEST_SUITE_P( XdsTest, XdsRbacTestWithActionAndAuditConditionPermutations, ::testing::Values( XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_ALLOW) .set_rbac_audit_condition( RBAC_AuditLoggingOptions_AuditCondition_ON_DENY) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_ALLOW) .set_rbac_audit_condition( RBAC_AuditLoggingOptions_AuditCondition_ON_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_ALLOW) .set_rbac_audit_condition( RBAC_AuditLoggingOptions_AuditCondition_ON_DENY_AND_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_DENY) .set_rbac_audit_condition( RBAC_AuditLoggingOptions_AuditCondition_ON_ALLOW) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_rbac_action(RBAC_Action_DENY) .set_rbac_audit_condition( RBAC_AuditLoggingOptions_AuditCondition_ON_DENY) .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar), XdsTestType() - .set_use_xds_credentials() .set_enable_rds_testing() .set_rbac_action(RBAC_Action_DENY) .set_rbac_audit_condition( diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index 9865a3a88de..63d089060ae 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -135,6 +135,15 @@ class XdsEnd2endTest::ServerThread::XdsChannelArgsServerBuilderOption // XdsEnd2endTest::ServerThread // +XdsEnd2endTest::ServerThread::ServerThread( + XdsEnd2endTest* test_obj, bool use_xds_enabled_server, + std::shared_ptr credentials) + : test_obj_(test_obj), + use_xds_enabled_server_(use_xds_enabled_server), + credentials_(credentials == nullptr ? CreateFakeServerCredentials() + : std::move(credentials)), + port_(grpc_pick_unused_port_or_die()) {} + void XdsEnd2endTest::ServerThread::Start() { LOG(INFO) << "starting " << Type() << " server on port " << port_; CHECK(!running_); @@ -198,7 +207,7 @@ void XdsEnd2endTest::ServerThread::Serve(grpc_core::Mutex* mu, builder.set_status_notifier(¬ifier_); builder.experimental().set_drain_grace_time( test_obj_->xds_drain_grace_time_ms_); - builder.AddListeningPort(server_address, Credentials()); + builder.AddListeningPort(server_address, credentials_); // Allow gRPC Core's HTTP server to accept PUT requests for testing // purposes. if (allow_put_requests_) { @@ -210,7 +219,7 @@ void XdsEnd2endTest::ServerThread::Serve(grpc_core::Mutex* mu, server_ = builder.BuildAndStart(); } else { ServerBuilder builder; - builder.AddListeningPort(server_address, Credentials()); + builder.AddListeningPort(server_address, credentials_); RegisterAllServices(&builder); server_ = builder.BuildAndStart(); } @@ -222,8 +231,9 @@ void XdsEnd2endTest::ServerThread::Serve(grpc_core::Mutex* mu, // XdsEnd2endTest::BackendServerThread::BackendServerThread( - XdsEnd2endTest* test_obj, bool use_xds_enabled_server) - : ServerThread(test_obj, use_xds_enabled_server) { + XdsEnd2endTest* test_obj, bool use_xds_enabled_server, + std::shared_ptr credentials) + : ServerThread(test_obj, use_xds_enabled_server, std::move(credentials)) { if (use_xds_enabled_server) { test_obj->SetServerListenerNameAndRouteConfiguration( test_obj->balancer_.get(), test_obj->default_server_listener_, port(), @@ -231,36 +241,6 @@ XdsEnd2endTest::BackendServerThread::BackendServerThread( } } -std::shared_ptr -XdsEnd2endTest::BackendServerThread::Credentials() { - if (GetParam().use_xds_credentials()) { - if (use_xds_enabled_server()) { - // We are testing server's use of XdsServerCredentials - return XdsServerCredentials(InsecureServerCredentials()); - } else { - // We are testing client's use of XdsCredentials - std::string root_cert = grpc_core::testing::GetFileContents(kCaCertPath); - std::string identity_cert = - grpc_core::testing::GetFileContents(kServerCertPath); - std::string private_key = - grpc_core::testing::GetFileContents(kServerKeyPath); - std::vector identity_key_cert_pairs = { - {private_key, identity_cert}}; - auto certificate_provider = - std::make_shared( - root_cert, identity_key_cert_pairs); - grpc::experimental::TlsServerCredentialsOptions options( - certificate_provider); - options.watch_root_certs(); - options.watch_identity_key_cert_pairs(); - options.set_cert_request_type( - GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); - return grpc::experimental::TlsServerCredentials(options); - } - } - return ServerThread::Credentials(); -} - void XdsEnd2endTest::BackendServerThread::RegisterAllServices( ServerBuilder* builder) { server_metric_recorder_ = ServerMetricRecorder::Create(); @@ -288,8 +268,10 @@ void XdsEnd2endTest::BackendServerThread::ShutdownAllServices() { // XdsEnd2endTest::BalancerServerThread::BalancerServerThread( - XdsEnd2endTest* test_obj, absl::string_view debug_label) - : ServerThread(test_obj, /*use_xds_enabled_server=*/false), + XdsEnd2endTest* test_obj, absl::string_view debug_label, + std::shared_ptr credentials) + : ServerThread(test_obj, /*use_xds_enabled_server=*/false, + std::move(credentials)), ads_service_(new AdsServiceImpl( // First request must have node set with the right client // features. @@ -318,26 +300,6 @@ XdsEnd2endTest::BalancerServerThread::BalancerServerThread( }, debug_label)) {} -std::shared_ptr -XdsEnd2endTest::BalancerServerThread::Credentials() { - if (GetParam().xds_server_uses_tls_creds()) { - std::string identity_cert = - grpc_core::testing::GetFileContents(kServerCertPath); - std::string private_key = - grpc_core::testing::GetFileContents(kServerKeyPath); - std::vector identity_key_cert_pairs = { - {private_key, identity_cert}}; - auto certificate_provider = - std::make_shared( - identity_key_cert_pairs); - grpc::experimental::TlsServerCredentialsOptions options( - certificate_provider); - options.watch_identity_key_cert_pairs(); - return grpc::experimental::TlsServerCredentials(options); - } - return ServerThread::Credentials(); -} - void XdsEnd2endTest::BalancerServerThread::RegisterAllServices( ServerBuilder* builder) { builder->RegisterService(ads_service_.get()); @@ -402,8 +364,10 @@ const char XdsEnd2endTest::kServerKeyPath[] = const char XdsEnd2endTest::kRequestMessage[] = "Live long and prosper."; -XdsEnd2endTest::XdsEnd2endTest() - : balancer_(CreateAndStartBalancer("Default Balancer")) { +XdsEnd2endTest::XdsEnd2endTest( + std::shared_ptr balancer_credentials) + : balancer_(CreateAndStartBalancer("Default Balancer", + std::move(balancer_credentials))) { // Initialize default client-side xDS resources. default_listener_ = XdsResourceUtils::DefaultListener(); default_route_config_ = XdsResourceUtils::DefaultRouteConfig(); @@ -435,9 +399,12 @@ void XdsEnd2endTest::TearDown() { } std::unique_ptr -XdsEnd2endTest::CreateAndStartBalancer(absl::string_view debug_label) { +XdsEnd2endTest::CreateAndStartBalancer( + absl::string_view debug_label, + std::shared_ptr credentials) { std::unique_ptr balancer = - std::make_unique(this, debug_label); + std::make_unique(this, debug_label, + std::move(credentials)); balancer->Start(); return balancer; } @@ -508,10 +475,12 @@ std::vector XdsEnd2endTest::GetBackendPorts(size_t start_index, return backend_ports; } -void XdsEnd2endTest::InitClient(absl::optional builder, - std::string lb_expected_authority, - int xds_resource_does_not_exist_timeout_ms, - std::string balancer_authority_override) { +void XdsEnd2endTest::InitClient( + absl::optional builder, + std::string lb_expected_authority, + int xds_resource_does_not_exist_timeout_ms, + std::string balancer_authority_override, + std::shared_ptr credentials) { if (!builder.has_value()) { builder = MakeBootstrapBuilder(); } @@ -560,12 +529,15 @@ void XdsEnd2endTest::InitClient(absl::optional builder, grpc_core::internal::UnsetGlobalXdsClientsForTest(); } // Create channel and stub. - ResetStub(); + ResetStub(/*failover_timeout_ms=*/0, /*args=*/nullptr, + std::move(credentials)); } -void XdsEnd2endTest::ResetStub(int failover_timeout_ms, - ChannelArguments* args) { - channel_ = CreateChannel(failover_timeout_ms, kServerName, "", args); +void XdsEnd2endTest::ResetStub( + int failover_timeout_ms, ChannelArguments* args, + std::shared_ptr credentials) { + channel_ = CreateChannel(failover_timeout_ms, kServerName, "", args, + std::move(credentials)); stub_ = grpc::testing::EchoTestService::NewStub(channel_); stub1_ = grpc::testing::EchoTest1Service::NewStub(channel_); stub2_ = grpc::testing::EchoTest2Service::NewStub(channel_); @@ -573,7 +545,7 @@ void XdsEnd2endTest::ResetStub(int failover_timeout_ms, std::shared_ptr XdsEnd2endTest::CreateChannel( int failover_timeout_ms, const char* server_name, const char* xds_authority, - ChannelArguments* args) { + ChannelArguments* args, std::shared_ptr credentials) { ChannelArguments local_args; if (args == nullptr) args = &local_args; // TODO(roth): Remove this once we enable retries by default internally. @@ -592,6 +564,7 @@ std::shared_ptr XdsEnd2endTest::CreateChannel( GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS, &xds_channel_args_, &kChannelArgsArgVtable); } + // Construct target URI. std::vector parts = {"xds:"}; if (xds_authority != nullptr && xds_authority[0] != '\0') { parts.emplace_back("//"); @@ -600,11 +573,11 @@ std::shared_ptr XdsEnd2endTest::CreateChannel( } parts.emplace_back(server_name); std::string uri = absl::StrJoin(parts, ""); - std::shared_ptr channel_creds = - GetParam().use_xds_credentials() - ? XdsCredentials(CreateTlsFallbackCredentials()) - : std::make_shared(); - return grpc::CreateCustomChannel(uri, channel_creds, *args); + // Credentials defaults to fake credentials. + if (credentials == nullptr) { + credentials = std::make_shared(); + } + return grpc::CreateCustomChannel(uri, credentials, *args); } Status XdsEnd2endTest::SendRpc( @@ -862,18 +835,19 @@ grpc_core::PemKeyCertPairList XdsEnd2endTest::ReadTlsIdentityPair( grpc_core::testing::GetFileContents(cert_path))}; } -std::shared_ptr -XdsEnd2endTest::CreateTlsFallbackCredentials() { - IdentityKeyCertPair key_cert_pair; - key_cert_pair.private_key = - grpc_core::testing::GetFileContents(kServerKeyPath); - key_cert_pair.certificate_chain = +std::vector +XdsEnd2endTest::MakeIdentityKeyCertPairForTlsCreds() { + std::string identity_cert = grpc_core::testing::GetFileContents(kServerCertPath); - std::vector identity_key_cert_pairs; - identity_key_cert_pairs.emplace_back(key_cert_pair); + std::string private_key = grpc_core::testing::GetFileContents(kServerKeyPath); + return {{std::move(private_key), std::move(identity_cert)}}; +} + +std::shared_ptr +XdsEnd2endTest::CreateXdsChannelCredentials() { auto certificate_provider = std::make_shared( grpc_core::testing::GetFileContents(kCaCertPath), - identity_key_cert_pairs); + MakeIdentityKeyCertPairForTlsCreds()); grpc::experimental::TlsChannelCredentialsOptions options; options.set_certificate_provider(std::move(certificate_provider)); options.watch_root_certs(); @@ -883,9 +857,39 @@ XdsEnd2endTest::CreateTlsFallbackCredentials() { options.set_certificate_verifier(std::move(verifier)); options.set_verify_server_certs(true); options.set_check_call_host(false); - auto channel_creds = grpc::experimental::TlsCredentials(options); - CHECK_NE(channel_creds.get(), nullptr); - return channel_creds; + auto tls_creds = grpc::experimental::TlsCredentials(options); + return XdsCredentials(tls_creds); +} + +std::shared_ptr +XdsEnd2endTest::CreateFakeServerCredentials() { + return std::make_shared( + grpc_fake_transport_security_server_credentials_create()); +} + +std::shared_ptr +XdsEnd2endTest::CreateMtlsServerCredentials() { + std::string root_cert = grpc_core::testing::GetFileContents(kCaCertPath); + auto certificate_provider = + std::make_shared( + std::move(root_cert), MakeIdentityKeyCertPairForTlsCreds()); + grpc::experimental::TlsServerCredentialsOptions options( + std::move(certificate_provider)); + options.watch_root_certs(); + options.watch_identity_key_cert_pairs(); + options.set_cert_request_type(GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); + return grpc::experimental::TlsServerCredentials(options); +} + +std::shared_ptr +XdsEnd2endTest::CreateTlsServerCredentials() { + auto certificate_provider = + std::make_shared( + MakeIdentityKeyCertPairForTlsCreds()); + grpc::experimental::TlsServerCredentialsOptions options( + std::move(certificate_provider)); + options.watch_identity_key_cert_pairs(); + return grpc::experimental::TlsServerCredentials(options); } } // namespace testing diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index c16aa278d1b..8b9c513d879 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -83,11 +83,6 @@ class XdsTestType { return *this; } - XdsTestType& set_use_xds_credentials() { - use_xds_credentials_ = true; - return *this; - } - XdsTestType& set_use_csds_streaming() { use_csds_streaming_ = true; return *this; @@ -115,14 +110,8 @@ class XdsTestType { return *this; } - XdsTestType& set_xds_server_uses_tls_creds(bool value) { - xds_server_uses_tls_creds_ = value; - return *this; - } - bool enable_load_reporting() const { return enable_load_reporting_; } bool enable_rds_testing() const { return enable_rds_testing_; } - bool use_xds_credentials() const { return use_xds_credentials_; } bool use_csds_streaming() const { return use_csds_streaming_; } HttpFilterConfigLocation filter_config_setup() const { return filter_config_setup_; @@ -135,13 +124,11 @@ class XdsTestType { rbac_audit_condition() const { return rbac_audit_condition_; } - bool xds_server_uses_tls_creds() const { return xds_server_uses_tls_creds_; } std::string AsString() const { std::string retval = "V3"; if (enable_load_reporting_) retval += "WithLoadReporting"; if (enable_rds_testing_) retval += "Rds"; - if (use_xds_credentials_) retval += "XdsCreds"; if (use_csds_streaming_) retval += "CsdsStreaming"; if (filter_config_setup_ == kHttpFilterConfigInRoute) { retval += "FilterPerRouteOverride"; @@ -164,9 +151,6 @@ class XdsTestType { RBAC_AuditLoggingOptions_AuditCondition_Name( rbac_audit_condition_)); } - if (xds_server_uses_tls_creds_) { - retval += "XdsServerUsesTlsCreds"; - } return retval; } @@ -178,7 +162,6 @@ class XdsTestType { private: bool enable_load_reporting_ = false; bool enable_rds_testing_ = false; - bool use_xds_credentials_ = false; bool use_csds_streaming_ = false; HttpFilterConfigLocation filter_config_setup_ = kHttpFilterConfigInListener; BootstrapSource bootstrap_source_ = kBootstrapFromChannelArg; @@ -187,7 +170,6 @@ class XdsTestType { ::envoy::config::rbac::v3::RBAC_AuditLoggingOptions_AuditCondition rbac_audit_condition_ = ::envoy::config::rbac::v3:: RBAC_AuditLoggingOptions_AuditCondition_NONE; - bool xds_server_uses_tls_creds_ = false; }; // A base class for xDS end-to-end tests. @@ -245,11 +227,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, }; // If use_xds_enabled_server is true, the server will use xDS. - explicit ServerThread(XdsEnd2endTest* test_obj, - bool use_xds_enabled_server = false) - : test_obj_(test_obj), - port_(grpc_pick_unused_port_or_die()), - use_xds_enabled_server_(use_xds_enabled_server) {} + // If credentials is null, fake credentials will be used. + explicit ServerThread( + XdsEnd2endTest* test_obj, bool use_xds_enabled_server = false, + std::shared_ptr credentials = nullptr); virtual ~ServerThread() { // Shutdown should be called manually. Shutdown calls virtual methods and @@ -260,17 +241,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, void Start(); void Shutdown(); - virtual std::shared_ptr Credentials() { - return std::make_shared( - grpc_fake_transport_security_server_credentials_create()); - } - std::string target() const { return absl::StrCat("localhost:", port_); } int port() const { return port_; } - bool use_xds_enabled_server() const { return use_xds_enabled_server_; } - XdsServingStatusNotifier* notifier() { return ¬ifier_; } void set_allow_put_requests(bool allow_put_requests) { @@ -292,12 +266,13 @@ class XdsEnd2endTest : public ::testing::TestWithParam, void Serve(grpc_core::Mutex* mu, grpc_core::CondVar* cond); XdsEnd2endTest* test_obj_; + const bool use_xds_enabled_server_; + const std::shared_ptr credentials_; const int port_; std::unique_ptr server_; XdsServingStatusNotifier notifier_; std::unique_ptr thread_; bool running_ = false; - const bool use_xds_enabled_server_; bool allow_put_requests_ = false; }; @@ -373,7 +348,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam, }; // If use_xds_enabled_server is true, the server will use xDS. - BackendServerThread(XdsEnd2endTest* test_obj, bool use_xds_enabled_server); + BackendServerThread(XdsEnd2endTest* test_obj, bool use_xds_enabled_server, + std::shared_ptr credentials); BackendServiceImpl* backend_service() { @@ -391,13 +367,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam, return server_metric_recorder_.get(); } - // If XdsTestType::use_xds_credentials() and use_xds_enabled_server() - // are both true, returns XdsServerCredentials. - // Otherwise, if XdsTestType::use_xds_credentials() is true and - // use_xds_enabled_server() is false, returns TlsServerCredentials. - // Otherwise, returns fake credentials. - std::shared_ptr Credentials() override; - private: const char* Type() override { return "Backend"; } void RegisterAllServices(ServerBuilder* builder) override; @@ -416,17 +385,13 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // A server thread for the xDS server. class BalancerServerThread : public ServerThread { public: - explicit BalancerServerThread(XdsEnd2endTest* test_obj, - absl::string_view debug_label); + explicit BalancerServerThread( + XdsEnd2endTest* test_obj, absl::string_view debug_label, + std::shared_ptr credentials); AdsServiceImpl* ads_service() { return ads_service_.get(); } LrsServiceImpl* lrs_service() { return lrs_service_.get(); } - // If XdsTestType::xds_server_uses_tls_creds() is true, - // returns TlsServerCredentials. - // Otherwise, returns fake credentials. - std::shared_ptr Credentials() override; - private: const char* Type() override { return "Balancer"; } void RegisterAllServices(ServerBuilder* builder) override; @@ -451,7 +416,9 @@ class XdsEnd2endTest : public ::testing::TestWithParam, METHOD_ECHO2, }; - XdsEnd2endTest(); + // If balancer_credentials is null, it defaults to fake credentials. + explicit XdsEnd2endTest( + std::shared_ptr balancer_credentials = nullptr); void SetUp() override { InitClient(); } void TearDown() override; @@ -463,8 +430,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // Creates and starts a new balancer, running in its own thread. // Most tests will not need to call this; instead, they can use // balancer_, which is already populated with default resources. + // If credentials is null, it defaults to fake credentials. std::unique_ptr CreateAndStartBalancer( - absl::string_view debug_label = ""); + absl::string_view debug_label = "", + std::shared_ptr credentials = nullptr); // Sets the Listener and RouteConfiguration resource on the specified // balancer. If RDS is in use, they will be set as separate resources; @@ -535,12 +504,15 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // Backend management // - // Creates num_backends backends and stores them in backends_, but - // does not actually start them. If xds_enabled is true, the backends - // are xDS-enabled. - void CreateBackends(size_t num_backends, bool xds_enabled = false) { + // Creates num_backends backends and stores them in backends_, but does + // not actually start them. If xds_enabled is true, the backends are + // xDS-enabled. If credentials is null, it defaults to fake credentials. + void CreateBackends( + size_t num_backends, bool xds_enabled = false, + std::shared_ptr credentials = nullptr) { for (size_t i = 0; i < num_backends; ++i) { - backends_.emplace_back(new BackendServerThread(this, xds_enabled)); + backends_.emplace_back( + new BackendServerThread(this, xds_enabled, credentials)); } } @@ -550,8 +522,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, } // Same as CreateBackends(), but also starts the backends. - void CreateAndStartBackends(size_t num_backends, bool xds_enabled = false) { - CreateBackends(num_backends, xds_enabled); + void CreateAndStartBackends( + size_t num_backends, bool xds_enabled = false, + std::shared_ptr credentials = nullptr) { + CreateBackends(num_backends, xds_enabled, std::move(credentials)); StartAllBackends(); } @@ -591,10 +565,12 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // Initializes global state for the client, such as the bootstrap file // and channel args for the XdsClient. Then calls ResetStub(). // All tests must call this exactly once at the start of the test. + // If credentials is null, fake credentials will be used. void InitClient(absl::optional builder = absl::nullopt, std::string lb_expected_authority = "", int xds_resource_does_not_exist_timeout_ms = 0, - std::string balancer_authority_override = ""); + std::string balancer_authority_override = "", + std::shared_ptr credentials = nullptr); XdsBootstrapBuilder MakeBootstrapBuilder() { return XdsBootstrapBuilder().SetServers( @@ -602,14 +578,17 @@ class XdsEnd2endTest : public ::testing::TestWithParam, } // Sets channel_, stub_, stub1_, and stub2_. - void ResetStub(int failover_timeout_ms = 0, ChannelArguments* args = nullptr); + // If credentials is null, fake credentials will be used. + void ResetStub(int failover_timeout_ms = 0, ChannelArguments* args = nullptr, + std::shared_ptr credentials = nullptr); // Creates a new client channel. Requires that InitClient() has // already been called. - std::shared_ptr CreateChannel(int failover_timeout_ms = 0, - const char* server_name = kServerName, - const char* xds_authority = "", - ChannelArguments* args = nullptr); + // If credentials is null, fake credentials will be used. + std::shared_ptr CreateChannel( + int failover_timeout_ms = 0, const char* server_name = kServerName, + const char* xds_authority = "", ChannelArguments* args = nullptr, + std::shared_ptr credentials = nullptr); // // Sending RPCs @@ -976,9 +955,18 @@ class XdsEnd2endTest : public ::testing::TestWithParam, static grpc_core::PemKeyCertPairList ReadTlsIdentityPair( const char* key_path, const char* cert_path); - // Returns client credentials suitable for using as fallback - // credentials for XdsCredentials. - static std::shared_ptr CreateTlsFallbackCredentials(); + // Internal helper function for creating TLS and mTLS creds. + // Not intended to be used by tests. + static std::vector + MakeIdentityKeyCertPairForTlsCreds(); + + // Returns XdsCredentials with mTLS fallback creds. + static std::shared_ptr CreateXdsChannelCredentials(); + + // Creates various types of server credentials. + static std::shared_ptr CreateFakeServerCredentials(); + static std::shared_ptr CreateMtlsServerCredentials(); + static std::shared_ptr CreateTlsServerCredentials(); std::unique_ptr balancer_;