From ac4f4de19626a11acb907053d85d71725967231f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 14 Jan 2021 14:27:51 -0800 Subject: [PATCH] xDS server config fetcher --- .../client_channel/lb_policy/xds/cds.cc | 11 - .../transport/chttp2/server/chttp2_server.cc | 69 +- .../transport/chttp2/server/chttp2_server.h | 13 +- .../chttp2/server/insecure/server_chttp2.cc | 12 +- .../server/secure/server_secure_chttp2.cc | 80 +- src/core/ext/xds/xds_api.cc | 457 +++++++---- src/core/ext/xds/xds_api.h | 23 +- src/core/ext/xds/xds_certificate_provider.cc | 22 +- src/core/ext/xds/xds_certificate_provider.h | 20 +- src/core/ext/xds/xds_client.cc | 11 +- src/core/ext/xds/xds_server_config_fetcher.cc | 158 +++- .../lib/http/httpcli_security_connector.cc | 4 +- .../credentials/alts/alts_credentials.cc | 3 +- .../credentials/alts/alts_credentials.h | 2 +- .../lib/security/credentials/credentials.h | 3 +- .../credentials/fake/fake_credentials.cc | 2 +- .../insecure/insecure_credentials.cc | 4 +- .../credentials/local/local_credentials.cc | 3 +- .../credentials/local/local_credentials.h | 2 +- .../credentials/ssl/ssl_credentials.cc | 3 +- .../credentials/ssl/ssl_credentials.h | 2 +- .../credentials/tls/tls_credentials.cc | 3 +- .../credentials/tls/tls_credentials.h | 2 +- .../credentials/xds/xds_credentials.cc | 32 +- .../credentials/xds/xds_credentials.h | 4 +- src/core/lib/surface/server.h | 3 + src/proto/grpc/testing/xds/v3/BUILD | 4 + src/proto/grpc/testing/xds/v3/listener.proto | 55 ++ src/proto/grpc/testing/xds/v3/tls.proto | 12 + .../core/end2end/h2_ssl_session_reuse_test.cc | 1 - .../grpc_tls_credentials_options_test.cc | 14 +- test/core/security/ssl_server_fuzzer.cc | 2 +- .../security/tls_security_connector_test.cc | 8 +- test/cpp/end2end/xds_end2end_test.cc | 744 ++++++++++++++++-- 34 files changed, 1462 insertions(+), 326 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc index 6a49d399eb8..d28f5e64295 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc @@ -270,17 +270,6 @@ void CdsLb::ShutdownLocked() { } xds_client_.reset(DEBUG_LOCATION, "CdsLb"); } - if (xds_certificate_provider_ != nullptr) { - // Unregister root and identity distributors from xDS provider, so - // that we don't leak memory. - // TODO(yashkt): This should not be necessary. Consider changing - // the provider API to use DualRefCounted<> instead of RefCounted<>, - // so that the xDS provider can use weak refs internally. - xds_certificate_provider_->UpdateRootCertNameAndDistributor( - config_->cluster(), "", nullptr); - xds_certificate_provider_->UpdateIdentityCertNameAndDistributor( - config_->cluster(), "", nullptr); - } grpc_channel_args_destroy(args_); args_ = nullptr; } diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index 8446a9f9189..e930407f3f1 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -62,13 +62,17 @@ const char kUnixAbstractUriPrefix[] = "unix-abstract:"; class Chttp2ServerListener : public Server::ListenerInterface { public: static grpc_error* Create(Server* server, grpc_resolved_address* addr, - grpc_channel_args* args, int* port_num); + grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier, + int* port_num); static grpc_error* CreateWithAcceptor(Server* server, const char* name, - grpc_channel_args* args); + grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier); // Do not instantiate directly. Use one of the factory methods above. - Chttp2ServerListener(Server* server, grpc_channel_args* args); + Chttp2ServerListener(Server* server, grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier); ~Chttp2ServerListener() override; void Start(Server* server, @@ -92,9 +96,15 @@ class Chttp2ServerListener : public Server::ListenerInterface { void UpdateConfig(grpc_channel_args* args) override { { MutexLock lock(&listener_->mu_); - // TODO(yashykt): Fix this - // grpc_channel_args_destroy(listener_->args_); - // listener_->args_ = args; + grpc_channel_args_destroy(listener_->args_); + grpc_error* error = GRPC_ERROR_NONE; + args = listener_->args_modifier_(args, &error); + if (error != GRPC_ERROR_NONE) { + // TODO(yashykt): Set state to close down connections immediately + // after accepting. + GPR_ASSERT(0); + } + listener_->args_ = args; if (!listener_->shutdown_) return; // Already started listening. } int port_temp; @@ -157,10 +167,11 @@ class Chttp2ServerListener : public Server::ListenerInterface { grpc_closure* destroy_done); Server* const server_; - grpc_channel_args* const args_; grpc_tcp_server* tcp_server_; grpc_resolved_address resolved_address_; + Chttp2ServerArgsModifier args_modifier_; Mutex mu_; + grpc_channel_args* args_; // guarded by mu_ ConfigFetcherWatcher* config_fetcher_watcher_ = nullptr; bool shutdown_ = true; grpc_closure tcp_server_shutdown_complete_; @@ -328,13 +339,14 @@ void Chttp2ServerListener::ConnectionState::OnHandshakeDone(void* arg, grpc_error* Chttp2ServerListener::Create(Server* server, grpc_resolved_address* addr, grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier, int* port_num) { Chttp2ServerListener* listener = nullptr; // The bulk of this method is inside of a lambda to make cleanup // easier without using goto. grpc_error* error = [&]() { // Create Chttp2ServerListener. - listener = new Chttp2ServerListener(server, args); + listener = new Chttp2ServerListener(server, args, args_modifier); error = grpc_tcp_server_create(&listener->tcp_server_shutdown_complete_, args, &listener->tcp_server_); if (error != GRPC_ERROR_NONE) return error; @@ -374,10 +386,11 @@ grpc_error* Chttp2ServerListener::Create(Server* server, return error; } -grpc_error* Chttp2ServerListener::CreateWithAcceptor(Server* server, - const char* name, - grpc_channel_args* args) { - Chttp2ServerListener* listener = new Chttp2ServerListener(server, args); +grpc_error* Chttp2ServerListener::CreateWithAcceptor( + Server* server, const char* name, grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier) { + Chttp2ServerListener* listener = + new Chttp2ServerListener(server, args, args_modifier); grpc_error* error = grpc_tcp_server_create( &listener->tcp_server_shutdown_complete_, args, &listener->tcp_server_); if (error != GRPC_ERROR_NONE) { @@ -392,9 +405,10 @@ grpc_error* Chttp2ServerListener::CreateWithAcceptor(Server* server, return GRPC_ERROR_NONE; } -Chttp2ServerListener::Chttp2ServerListener(Server* server, - grpc_channel_args* args) - : server_(server), args_(args) { +Chttp2ServerListener::Chttp2ServerListener( + Server* server, grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier) + : server_(server), args_modifier_(args_modifier), args_(args) { GRPC_CLOSURE_INIT(&tcp_server_shutdown_complete_, TcpServerShutdownComplete, this, grpc_schedule_on_exec_ctx); } @@ -407,13 +421,16 @@ Chttp2ServerListener::~Chttp2ServerListener() { void Chttp2ServerListener::Start( Server* /*server*/, const std::vector* /* pollsets */) { if (server_->config_fetcher() != nullptr) { + grpc_channel_args* args = nullptr; auto watcher = absl::make_unique(this); { MutexLock lock(&mu_); config_fetcher_watcher_ = watcher.get(); + args = grpc_channel_args_copy(args_); } server_->config_fetcher()->StartWatch( - grpc_sockaddr_to_string(&resolved_address_, false), std::move(watcher)); + grpc_sockaddr_to_string(&resolved_address_, false), args, + std::move(watcher)); } else { StartListening(); } @@ -459,9 +476,15 @@ void Chttp2ServerListener::OnAccept(void* arg, grpc_endpoint* tcp, gpr_free(acceptor); return; } + grpc_channel_args* args = nullptr; + { + MutexLock lock(&self->mu_); + args = grpc_channel_args_copy(self->args_); + } // Deletes itself when done. new ConnectionState(self, accepting_pollset, acceptor, - std::move(handshake_mgr), self->args_, tcp); + std::move(handshake_mgr), args, tcp); + grpc_channel_args_destroy(args); } void Chttp2ServerListener::TcpServerShutdownComplete(void* arg, @@ -513,10 +536,12 @@ void Chttp2ServerListener::Orphan() { // grpc_error* Chttp2ServerAddPort(Server* server, const char* addr, - grpc_channel_args* args, int* port_num) { + grpc_channel_args* args, + Chttp2ServerArgsModifier args_modifier, + int* port_num) { if (strncmp(addr, "external:", 9) == 0) { - return grpc_core::Chttp2ServerListener::CreateWithAcceptor(server, addr, - args); + return grpc_core::Chttp2ServerListener::CreateWithAcceptor( + server, addr, args, args_modifier); } *port_num = -1; grpc_resolved_addresses* resolved = nullptr; @@ -540,10 +565,10 @@ grpc_error* Chttp2ServerAddPort(Server* server, const char* addr, if (*port_num != -1 && grpc_sockaddr_get_port(&resolved->addrs[i]) == 0) { grpc_sockaddr_set_port(&resolved->addrs[i], *port_num); } - int port_temp; + int port_temp = -1; error = grpc_core::Chttp2ServerListener::Create( server, &resolved->addrs[i], grpc_channel_args_copy(args), - &port_temp); + args_modifier, &port_temp); if (error != GRPC_ERROR_NONE) { error_list.push_back(error); } else { diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.h b/src/core/ext/transport/chttp2/server/chttp2_server.h index 095cc5e4c04..10322e9af34 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.h +++ b/src/core/ext/transport/chttp2/server/chttp2_server.h @@ -28,10 +28,19 @@ namespace grpc_core { +// A function to modify channel args for a listening addr:port. Note that this +// is used to create a security connector for listeners when the servers are +// configured with a config fetcher. Not invoked if there is no config fetcher +// added to the server. Takes ownership of the args. Caller takes ownership of +// returned args. On failure, the error parameter will be set. +using Chttp2ServerArgsModifier = + std::function; + /// Adds a port to \a server. Sets \a port_num to the port number. /// Takes ownership of \a args. -grpc_error* Chttp2ServerAddPort(Server* server, const char* addr, - grpc_channel_args* args, int* port_num); +grpc_error* Chttp2ServerAddPort( + Server* server, const char* addr, grpc_channel_args* args, + Chttp2ServerArgsModifier connection_args_modifier, int* port_num); } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc index eebae9efaa9..4980b4eaeb7 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc @@ -27,6 +27,15 @@ #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" +namespace { + +grpc_channel_args* ModifyArgsForConnection(grpc_channel_args* args, + grpc_error** error) { + return args; +} + +} // namespace + int grpc_server_add_insecure_http2_port(grpc_server* server, const char* addr) { grpc_core::ExecCtx exec_ctx; int port_num = 0; @@ -34,7 +43,8 @@ int grpc_server_add_insecure_http2_port(grpc_server* server, const char* addr) { (server, addr)); grpc_error* err = grpc_core::Chttp2ServerAddPort( server->core_server.get(), addr, - grpc_channel_args_copy(server->core_server->channel_args()), &port_num); + grpc_channel_args_copy(server->core_server->channel_args()), + ModifyArgsForConnection, &port_num); if (err != GRPC_ERROR_NONE) { const char* msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc index a181db47f39..563c3034902 100644 --- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc +++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc @@ -18,12 +18,11 @@ #include -#include - #include #include "absl/strings/str_cat.h" +#include #include #include @@ -38,6 +37,35 @@ #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" +namespace { + +grpc_channel_args* ModifyArgsForConnection(grpc_channel_args* args, + grpc_error** error) { + grpc_server_credentials* server_credentials = + grpc_find_server_credentials_in_args(args); + if (server_credentials == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Could not find server credentials"); + return args; + } + auto security_connector = server_credentials->create_security_connector(args); + if (security_connector == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("Unable to create secure server with credentials of type ", + server_credentials->type()) + .c_str()); + return args; + } + grpc_arg arg_to_add = + grpc_security_connector_to_arg(security_connector.get()); + grpc_channel_args* new_args = + grpc_channel_args_copy_and_add(args, &arg_to_add, 1); + grpc_channel_args_destroy(args); + return new_args; +} + +} // namespace + int grpc_server_add_secure_http2_port(grpc_server* server, const char* addr, grpc_server_credentials* creds) { grpc_core::ExecCtx exec_ctx; @@ -55,27 +83,43 @@ int grpc_server_add_secure_http2_port(grpc_server* server, const char* addr, "No credentials specified for secure server port (creds==NULL)"); goto done; } - sc = creds->create_security_connector(); - if (sc == nullptr) { - err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("Unable to create secure server with credentials of type ", - creds->type()) - .c_str()); - goto done; + // TODO(yashykt): Ideally, we would not want to have different behavior here + // based on whether a config fetcher is configured or not. Currently, we have + // a feature for SSL credentials reloading with an application callback that + // assumes that there is a single security connector. If we delay the creation + // of the security connector to after the creation of the listener(s), we + // would have potentially multiple security connectors which breaks the + // assumption for SSL creds reloading. When the API for SSL creds reloading is + // rewritten, we would be able to make this workaround go away by removing + // that assumption. As an immediate drawback of this workaround, config + // fetchers need to be registered before adding ports to the server. + if (server->core_server->config_fetcher() != nullptr) { + // Create channel args. + grpc_arg arg_to_add = grpc_server_credentials_to_arg(creds); + args = grpc_channel_args_copy_and_add(server->core_server->channel_args(), + &arg_to_add, 1); + } else { + sc = creds->create_security_connector(nullptr); + if (sc == nullptr) { + err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + "Unable to create secure server with credentials of type ", + creds->type()) + .c_str()); + goto done; + } + grpc_arg args_to_add[2]; + args_to_add[0] = grpc_server_credentials_to_arg(creds); + args_to_add[1] = grpc_security_connector_to_arg(sc.get()); + args = grpc_channel_args_copy_and_add(server->core_server->channel_args(), + args_to_add, + GPR_ARRAY_SIZE(args_to_add)); } - // Create channel args. - grpc_arg args_to_add[2]; - args_to_add[0] = grpc_server_credentials_to_arg(creds); - args_to_add[1] = grpc_security_connector_to_arg(sc.get()); - args = - grpc_channel_args_copy_and_add(server->core_server->channel_args(), - args_to_add, GPR_ARRAY_SIZE(args_to_add)); // Add server port. err = grpc_core::Chttp2ServerAddPort(server->core_server.get(), addr, args, - &port_num); + ModifyArgsForConnection, &port_num); done: sc.reset(DEBUG_LOCATION, "server"); - if (err != GRPC_ERROR_NONE) { const char* msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); diff --git a/src/core/ext/xds/xds_api.cc b/src/core/ext/xds/xds_api.cc index e9403c2c153..d1b172a3c54 100644 --- a/src/core/ext/xds/xds_api.cc +++ b/src/core/ext/xds/xds_api.cc @@ -39,6 +39,7 @@ #include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/slice/slice_utils.h" @@ -57,6 +58,7 @@ #include "envoy/config/endpoint/v3/load_report.upb.h" #include "envoy/config/listener/v3/api_listener.upb.h" #include "envoy/config/listener/v3/listener.upb.h" +#include "envoy/config/listener/v3/listener_components.upb.h" #include "envoy/config/route/v3/route.upb.h" #include "envoy/config/route/v3/route.upbdefs.h" #include "envoy/config/route/v3/route_components.upb.h" @@ -609,6 +611,45 @@ bool XdsApi::CommonTlsContext::Empty() const { combined_validation_context.Empty(); } +// +// XdsApi::DownstreamTlsContext +// + +std::string XdsApi::DownstreamTlsContext::ToString() const { + return absl::StrFormat("common_tls_context=%s, require_client_certificate=%s", + common_tls_context.ToString(), + require_client_certificate ? "true" : "false"); +} + +bool XdsApi::DownstreamTlsContext::Empty() const { + return common_tls_context.Empty(); +} + +// +// XdsApi::LdsUpdate +// + +std::string XdsApi::LdsUpdate::ToString() const { + absl::InlinedVector contents; + if (type == ListenerType::kTcpListener) { + if (!downstream_tls_context.Empty()) { + contents.push_back(absl::StrFormat("downstream_tls_context=%s", + downstream_tls_context.ToString())); + } + } else if (type == ListenerType::kHttpApiListener) { + contents.push_back(absl::StrFormat( + "route_config_name=%s", + !route_config_name.empty() ? route_config_name.c_str() : "")); + contents.push_back(absl::StrFormat("http_max_stream_duration=%s", + http_max_stream_duration.ToString())); + if (rds_update.has_value()) { + contents.push_back( + absl::StrFormat("rds_update=%s", rds_update->ToString())); + } + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); +} + // // XdsApi::CdsUpdate // @@ -1414,170 +1455,6 @@ grpc_error* RouteConfigParse( return GRPC_ERROR_NONE; } -grpc_error* LdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, - const envoy_service_discovery_v3_DiscoveryResponse* response, - const std::set& expected_listener_names, - XdsApi::LdsUpdateMap* lds_update_map, upb_arena* arena) { - // Get the resources from the response. - size_t size; - const google_protobuf_Any* const* resources = - envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); - for (size_t i = 0; i < size; ++i) { - // Check the type_url of the resource. - absl::string_view type_url = - UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); - if (!IsLds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not LDS."); - } - // Decode the listener. - const upb_strview encoded_listener = - google_protobuf_Any_value(resources[i]); - const envoy_config_listener_v3_Listener* listener = - envoy_config_listener_v3_Listener_parse(encoded_listener.data, - encoded_listener.size, arena); - if (listener == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode listener."); - } - // Check listener name. Ignore unexpected listeners. - std::string listener_name = - UpbStringToStdString(envoy_config_listener_v3_Listener_name(listener)); - if (expected_listener_names.find(listener_name) == - expected_listener_names.end()) { - continue; - } - // Fail if listener name is duplicated. - if (lds_update_map->find(listener_name) != lds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("duplicate listener name \"", listener_name, "\"") - .c_str()); - } - XdsApi::LdsUpdate& lds_update = (*lds_update_map)[listener_name]; - // Get api_listener and decode it to http_connection_manager. - const envoy_config_listener_v3_ApiListener* api_listener = - envoy_config_listener_v3_Listener_api_listener(listener); - if (api_listener == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Listener has no ApiListener."); - } - const upb_strview encoded_api_listener = google_protobuf_Any_value( - envoy_config_listener_v3_ApiListener_api_listener(api_listener)); - const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* - http_connection_manager = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( - encoded_api_listener.data, encoded_api_listener.size, arena); - if (http_connection_manager == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Could not parse HttpConnectionManager config from ApiListener"); - } - if (XdsTimeoutEnabled()) { - // Obtain max_stream_duration from Http Protocol Options. - const envoy_config_core_v3_HttpProtocolOptions* options = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_common_http_protocol_options( - http_connection_manager); - if (options != nullptr) { - const google_protobuf_Duration* duration = - envoy_config_core_v3_HttpProtocolOptions_max_stream_duration( - options); - if (duration != nullptr) { - lds_update.http_max_stream_duration.seconds = - google_protobuf_Duration_seconds(duration); - lds_update.http_max_stream_duration.nanos = - google_protobuf_Duration_nanos(duration); - } - } - } - // Found inlined route_config. Parse it to find the cluster_name. - if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( - http_connection_manager)) { - const envoy_config_route_v3_RouteConfiguration* route_config = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( - http_connection_manager); - XdsApi::RdsUpdate rds_update; - grpc_error* error = - RouteConfigParse(client, tracer, symtab, route_config, &rds_update); - if (error != GRPC_ERROR_NONE) return error; - lds_update.rds_update = std::move(rds_update); - continue; - } - // Validate that RDS must be used to get the route_config dynamically. - if (!envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_rds( - http_connection_manager)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager neither has inlined route_config nor RDS."); - } - const envoy_extensions_filters_network_http_connection_manager_v3_Rds* rds = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_rds( - http_connection_manager); - // Check that the ConfigSource specifies ADS. - const envoy_config_core_v3_ConfigSource* config_source = - envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( - rds); - if (config_source == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager missing config_source for RDS."); - } - if (!envoy_config_core_v3_ConfigSource_has_ads(config_source)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager ConfigSource for RDS does not specify ADS."); - } - // Get the route_config_name. - lds_update.route_config_name = UpbStringToStdString( - envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( - rds)); - } - return GRPC_ERROR_NONE; -} - -grpc_error* RdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, - const envoy_service_discovery_v3_DiscoveryResponse* response, - const std::set& expected_route_configuration_names, - XdsApi::RdsUpdateMap* rds_update_map, upb_arena* arena) { - // Get the resources from the response. - size_t size; - const google_protobuf_Any* const* resources = - envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); - for (size_t i = 0; i < size; ++i) { - // Check the type_url of the resource. - absl::string_view type_url = - UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); - if (!IsRds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not RDS."); - } - // Decode the route_config. - const upb_strview encoded_route_config = - google_protobuf_Any_value(resources[i]); - const envoy_config_route_v3_RouteConfiguration* route_config = - envoy_config_route_v3_RouteConfiguration_parse( - encoded_route_config.data, encoded_route_config.size, arena); - if (route_config == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode route_config."); - } - // Check route_config_name. Ignore unexpected route_config. - std::string route_config_name = UpbStringToStdString( - envoy_config_route_v3_RouteConfiguration_name(route_config)); - if (expected_route_configuration_names.find(route_config_name) == - expected_route_configuration_names.end()) { - continue; - } - // Fail if route config name is duplicated. - if (rds_update_map->find(route_config_name) != rds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("duplicate route config name \"", route_config_name, - "\"") - .c_str()); - } - // Parse the route_config. - XdsApi::RdsUpdate& rds_update = - (*rds_update_map)[std::move(route_config_name)]; - grpc_error* error = - RouteConfigParse(client, tracer, symtab, route_config, &rds_update); - if (error != GRPC_ERROR_NONE) return error; - } - return GRPC_ERROR_NONE; -} - XdsApi::CommonTlsContext::CertificateProviderInstance CertificateProviderInstanceParse( const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance* @@ -1688,6 +1565,258 @@ grpc_error* CommonTlsContextParse( return GRPC_ERROR_NONE; } +grpc_error* LdsResponseParseClient( + XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, upb_arena* arena, + const envoy_config_listener_v3_ApiListener* api_listener, + XdsApi::LdsUpdate* lds_update) { + lds_update->type = XdsApi::LdsUpdate::ListenerType::kHttpApiListener; + const upb_strview encoded_api_listener = google_protobuf_Any_value( + envoy_config_listener_v3_ApiListener_api_listener(api_listener)); + const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* + http_connection_manager = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( + encoded_api_listener.data, encoded_api_listener.size, arena); + if (http_connection_manager == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Could not parse HttpConnectionManager config from ApiListener"); + } + if (XdsTimeoutEnabled()) { + // Obtain max_stream_duration from Http Protocol Options. + const envoy_config_core_v3_HttpProtocolOptions* options = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_common_http_protocol_options( + http_connection_manager); + if (options != nullptr) { + const google_protobuf_Duration* duration = + envoy_config_core_v3_HttpProtocolOptions_max_stream_duration(options); + if (duration != nullptr) { + lds_update->http_max_stream_duration.seconds = + google_protobuf_Duration_seconds(duration); + lds_update->http_max_stream_duration.nanos = + google_protobuf_Duration_nanos(duration); + } + } + } + // Found inlined route_config. Parse it to find the cluster_name. + if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( + http_connection_manager)) { + const envoy_config_route_v3_RouteConfiguration* route_config = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( + http_connection_manager); + XdsApi::RdsUpdate rds_update; + grpc_error* error = + RouteConfigParse(client, tracer, symtab, route_config, &rds_update); + if (error != GRPC_ERROR_NONE) return error; + lds_update->rds_update = std::move(rds_update); + return GRPC_ERROR_NONE; + } + // Validate that RDS must be used to get the route_config dynamically. + if (!envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_rds( + http_connection_manager)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager neither has inlined route_config nor RDS."); + } + const envoy_extensions_filters_network_http_connection_manager_v3_Rds* rds = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_rds( + http_connection_manager); + // Check that the ConfigSource specifies ADS. + const envoy_config_core_v3_ConfigSource* config_source = + envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( + rds); + if (config_source == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager missing config_source for RDS."); + } + if (!envoy_config_core_v3_ConfigSource_has_ads(config_source)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager ConfigSource for RDS does not specify ADS."); + } + // Get the route_config_name. + lds_update->route_config_name = UpbStringToStdString( + envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( + rds)); + return GRPC_ERROR_NONE; +} + +grpc_error* LdsResponseParseServer( + upb_arena* arena, const envoy_config_listener_v3_Listener* listener, + const std::string& listener_name, + const envoy_config_core_v3_Address* address, + XdsApi::LdsUpdate* lds_update) { + lds_update->type = XdsApi::LdsUpdate::ListenerType::kTcpListener; + // TODO(yashykt): Support filter chain match. + // Right now, we are supporting and expecting only one entry in filter_chains. + size_t size = 0; + auto* filter_chains = + envoy_config_listener_v3_Listener_filter_chains(listener, &size); + if (size != 1) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Only one filter_chain supported."); + } + // Get the DownstreamTlsContext from the match + if (XdsSecurityEnabled()) { + auto* transport_socket = + envoy_config_listener_v3_FilterChain_transport_socket(filter_chains[0]); + if (transport_socket != nullptr) { + absl::string_view name = UpbStringToAbsl( + envoy_config_core_v3_TransportSocket_name(transport_socket)); + if (name == "envoy.transport_sockets.tls") { + auto* typed_config = + envoy_config_core_v3_TransportSocket_typed_config(transport_socket); + if (typed_config != nullptr) { + const upb_strview encoded_downstream_tls_context = + google_protobuf_Any_value(typed_config); + auto* downstream_tls_context = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_parse( + encoded_downstream_tls_context.data, + encoded_downstream_tls_context.size, arena); + if (downstream_tls_context == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Can't decode downstream tls context."); + } + auto* common_tls_context = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_common_tls_context( + downstream_tls_context); + if (common_tls_context != nullptr) { + grpc_error* error = CommonTlsContextParse( + common_tls_context, + &lds_update->downstream_tls_context.common_tls_context); + if (error != GRPC_ERROR_NONE) return error; + } + auto* require_client_certificate = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_client_certificate( + downstream_tls_context); + if (require_client_certificate != nullptr) { + lds_update->downstream_tls_context.require_client_certificate = + google_protobuf_BoolValue_value(require_client_certificate); + } + } + if (lds_update->downstream_tls_context.common_tls_context + .tls_certificate_certificate_provider_instance.instance_name + .empty()) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "TLS configuration provided but no " + "tls_certificate_certificate_provider_instance found."); + } + } + } + } + return GRPC_ERROR_NONE; +} + +grpc_error* LdsResponseParse( + XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const envoy_service_discovery_v3_DiscoveryResponse* response, + const std::set& expected_listener_names, + XdsApi::LdsUpdateMap* lds_update_map, upb_arena* arena) { + // Get the resources from the response. + size_t size; + const google_protobuf_Any* const* resources = + envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); + for (size_t i = 0; i < size; ++i) { + // Check the type_url of the resource. + absl::string_view type_url = + UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); + if (!IsLds(type_url)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not LDS."); + } + // Decode the listener. + const upb_strview encoded_listener = + google_protobuf_Any_value(resources[i]); + const envoy_config_listener_v3_Listener* listener = + envoy_config_listener_v3_Listener_parse(encoded_listener.data, + encoded_listener.size, arena); + if (listener == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode listener."); + } + // Check listener name. Ignore unexpected listeners. + std::string listener_name = + UpbStringToStdString(envoy_config_listener_v3_Listener_name(listener)); + if (expected_listener_names.find(listener_name) == + expected_listener_names.end()) { + continue; + } + // Fail if listener name is duplicated. + if (lds_update_map->find(listener_name) != lds_update_map->end()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("duplicate listener name \"", listener_name, "\"") + .c_str()); + } + XdsApi::LdsUpdate& lds_update = (*lds_update_map)[listener_name]; + // Check whether it's a client or server listener. + const envoy_config_listener_v3_ApiListener* api_listener = + envoy_config_listener_v3_Listener_api_listener(listener); + const envoy_config_core_v3_Address* address = + envoy_config_listener_v3_Listener_address(listener); + if (api_listener != nullptr && address != nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Listener has both address and ApiListener"); + } + if (api_listener == nullptr && address == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Listener has neither address nor ApiListener"); + } + grpc_error* error = GRPC_ERROR_NONE; + if (api_listener != nullptr) { + error = LdsResponseParseClient(client, tracer, symtab, arena, + api_listener, &lds_update); + } else { + error = LdsResponseParseServer(arena, listener, listener_name, address, + &lds_update); + } + if (error != GRPC_ERROR_NONE) return error; + } + return GRPC_ERROR_NONE; +} + +grpc_error* RdsResponseParse( + XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const envoy_service_discovery_v3_DiscoveryResponse* response, + const std::set& expected_route_configuration_names, + XdsApi::RdsUpdateMap* rds_update_map, upb_arena* arena) { + // Get the resources from the response. + size_t size; + const google_protobuf_Any* const* resources = + envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); + for (size_t i = 0; i < size; ++i) { + // Check the type_url of the resource. + absl::string_view type_url = + UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); + if (!IsRds(type_url)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not RDS."); + } + // Decode the route_config. + const upb_strview encoded_route_config = + google_protobuf_Any_value(resources[i]); + const envoy_config_route_v3_RouteConfiguration* route_config = + envoy_config_route_v3_RouteConfiguration_parse( + encoded_route_config.data, encoded_route_config.size, arena); + if (route_config == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode route_config."); + } + // Check route_config_name. Ignore unexpected route_config. + std::string route_config_name = UpbStringToStdString( + envoy_config_route_v3_RouteConfiguration_name(route_config)); + if (expected_route_configuration_names.find(route_config_name) == + expected_route_configuration_names.end()) { + continue; + } + // Fail if route config name is duplicated. + if (rds_update_map->find(route_config_name) != rds_update_map->end()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("duplicate route config name \"", route_config_name, + "\"") + .c_str()); + } + // Parse the route_config. + XdsApi::RdsUpdate& rds_update = + (*rds_update_map)[std::move(route_config_name)]; + grpc_error* error = + RouteConfigParse(client, tracer, symtab, route_config, &rds_update); + if (error != GRPC_ERROR_NONE) return error; + } + return GRPC_ERROR_NONE; +} + grpc_error* CdsResponseParse( XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, const envoy_service_discovery_v3_DiscoveryResponse* response, diff --git a/src/core/ext/xds/xds_api.h b/src/core/ext/xds/xds_api.h index f2a67072912..ecc94e3320e 100644 --- a/src/core/ext/xds/xds_api.h +++ b/src/core/ext/xds/xds_api.h @@ -264,9 +264,27 @@ class XdsApi { bool Empty() const; }; + struct DownstreamTlsContext { + CommonTlsContext common_tls_context; + bool require_client_certificate = false; + + bool operator==(const DownstreamTlsContext& other) const { + return common_tls_context == other.common_tls_context && + require_client_certificate == other.require_client_certificate; + } + + std::string ToString() const; + bool Empty() const; + }; + // TODO(roth): When we can use absl::variant<>, consider using that // here, to enforce the fact that only one of the two fields can be set. struct LdsUpdate { + enum class ListenerType { + kTcpListener = 0, + kHttpApiListener, + } type; + DownstreamTlsContext downstream_tls_context; // The name to use in the RDS request. std::string route_config_name; // Storing the Http Connection Manager Common Http Protocol Option @@ -277,10 +295,13 @@ class XdsApi { absl::optional rds_update; bool operator==(const LdsUpdate& other) const { - return route_config_name == other.route_config_name && + return downstream_tls_context == other.downstream_tls_context && + route_config_name == other.route_config_name && rds_update == other.rds_update && http_max_stream_duration == other.http_max_stream_duration; } + + std::string ToString() const; }; using LdsUpdateMap = std::map; diff --git a/src/core/ext/xds/xds_certificate_provider.cc b/src/core/ext/xds/xds_certificate_provider.cc index 0386be1cf6b..daeb6e422bc 100644 --- a/src/core/ext/xds/xds_certificate_provider.cc +++ b/src/core/ext/xds/xds_certificate_provider.cc @@ -280,7 +280,7 @@ void XdsCertificateProvider::UpdateRootCertNameAndDistributor( if (it == certificate_state_map_.end()) { it = certificate_state_map_ .emplace(cert_name, - absl::make_unique(Ref())) + absl::make_unique(this)) .first; } it->second->UpdateRootCertNameAndDistributor(cert_name, root_cert_name, @@ -305,7 +305,7 @@ void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( if (it == certificate_state_map_.end()) { it = certificate_state_map_ .emplace(cert_name, - absl::make_unique(Ref())) + absl::make_unique(this)) .first; } it->second->UpdateIdentityCertNameAndDistributor( @@ -314,6 +314,22 @@ void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); } +bool XdsCertificateProvider::GetRequireClientCertificate( + const std::string& cert_name) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return false; + return it->second->require_client_certificate(); +} + +void XdsCertificateProvider::UpdateRequireClientCertificate( + const std::string& cert_name, bool require_client_certificate) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return; + it->second->set_require_client_certificate(require_client_certificate); +} + std::vector XdsCertificateProvider::GetSanMatchers( const std::string& cluster) { MutexLock lock(&san_matchers_mu_); @@ -340,7 +356,7 @@ void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, if (it == certificate_state_map_.end()) { it = certificate_state_map_ .emplace(cert_name, - absl::make_unique(Ref())) + absl::make_unique(this)) .first; } it->second->WatchStatusCallback(cert_name, root_being_watched, diff --git a/src/core/ext/xds/xds_certificate_provider.h b/src/core/ext/xds/xds_certificate_provider.h index 6fbedb13b55..d61ae8b5a8b 100644 --- a/src/core/ext/xds/xds_certificate_provider.h +++ b/src/core/ext/xds/xds_certificate_provider.h @@ -50,6 +50,12 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { RefCountedPtr identity_cert_distributor); + bool GetRequireClientCertificate(const std::string& cert_name); + // Updating \a require_client_certificate for a non-existing \a cert_name has + // no effect. + void UpdateRequireClientCertificate(const std::string& cert_name, + bool require_client_certificate); + std::vector GetSanMatchers(const std::string& cluster); void UpdateSubjectAlternativeNameMatchers( const std::string& cluster, std::vector matchers); @@ -63,8 +69,8 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { class ClusterCertificateState { public: explicit ClusterCertificateState( - RefCountedPtr xds_certificate_provider) - : xds_certificate_provider_(std::move(xds_certificate_provider)) {} + XdsCertificateProvider* xds_certificate_provider) + : xds_certificate_provider_(xds_certificate_provider) {} ~ClusterCertificateState(); @@ -92,12 +98,19 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { const std::string& cert_name, grpc_tls_certificate_distributor* identity_cert_distributor); + bool require_client_certificate() const { + return require_client_certificate_; + } + void set_require_client_certificate(bool require_client_certificate) { + require_client_certificate_ = require_client_certificate; + } + void WatchStatusCallback(const std::string& cert_name, bool root_being_watched, bool identity_being_watched); private: - RefCountedPtr xds_certificate_provider_; + XdsCertificateProvider* xds_certificate_provider_; bool watching_root_certs_ = false; bool watching_identity_certs_ = false; std::string root_cert_name_; @@ -108,6 +121,7 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { root_cert_watcher_ = nullptr; grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* identity_cert_watcher_ = nullptr; + bool require_client_certificate_ = false; }; void WatchStatusCallback(std::string cert_name, bool root_being_watched, diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index e95cf7aa663..5b360eb0bdc 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -882,15 +882,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( auto& state = lds_state.subscribed_resources[listener_name]; if (state != nullptr) state->Finish(); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "[xds_client %p] LDS resource %s: route_config_name=%s", - xds_client(), listener_name.c_str(), - (!lds_update.route_config_name.empty() - ? lds_update.route_config_name.c_str() - : "")); - if (lds_update.rds_update.has_value()) { - gpr_log(GPR_INFO, "RouteConfiguration: %s", - lds_update.rds_update->ToString().c_str()); - } + gpr_log(GPR_INFO, "[xds_client %p] LDS resource %s: %s", xds_client(), + listener_name.c_str(), lds_update.ToString().c_str()); } // Record the RDS resource names seen. if (!lds_update.route_config_name.empty()) { diff --git a/src/core/ext/xds/xds_server_config_fetcher.cc b/src/core/ext/xds/xds_server_config_fetcher.cc index 5c5e8ee21f1..d1dae59e411 100644 --- a/src/core/ext/xds/xds_server_config_fetcher.cc +++ b/src/core/ext/xds/xds_server_config_fetcher.cc @@ -18,11 +18,18 @@ #include +#include "src/core/ext/xds/xds_certificate_provider.h" #include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/security/credentials/xds/xds_credentials.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" namespace grpc_core { + +TraceFlag grpc_xds_server_config_fetcher_trace(false, + "xds_server_config_fetcher"); + namespace { class XdsServerConfigFetcher : public grpc_server_config_fetcher { @@ -32,18 +39,18 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { GPR_ASSERT(xds_client_ != nullptr); } - void StartWatch(std::string listening_address, + void StartWatch(std::string listening_address, grpc_channel_args* args, std::unique_ptr watcher) override { grpc_server_config_fetcher::WatcherInterface* watcher_ptr = watcher.get(); - auto listener_watcher = - absl::make_unique(std::move(watcher)); + auto listener_watcher = absl::make_unique( + std::move(watcher), args, xds_client_); auto* listener_watcher_ptr = listener_watcher.get(); // TODO(yashykt): Get the resource name id from bootstrap - xds_client_->WatchListenerData( - absl::StrCat("grpc/server?xds.resource.listening_address=", - listening_address), - std::move(listener_watcher)); + listening_address = absl::StrCat( + "grpc/server?xds.resource.listening_address=", listening_address); + xds_client_->WatchListenerData(listening_address, + std::move(listener_watcher)); MutexLock lock(&mu_); auto& watcher_state = watchers_[watcher_ptr]; watcher_state.listening_address = listening_address; @@ -73,12 +80,45 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { public: explicit ListenerWatcher( std::unique_ptr - server_config_watcher) - : server_config_watcher_(std::move(server_config_watcher)) {} + server_config_watcher, + grpc_channel_args* args, RefCountedPtr xds_client) + : server_config_watcher_(std::move(server_config_watcher)), + args_(args), + xds_client_(std::move(xds_client)) {} + + ~ListenerWatcher() override { grpc_channel_args_destroy(args_); } + + // Deleted due to special handling required for args_. Copy the channel args + // if we ever need these. + ListenerWatcher(const ListenerWatcher&) = delete; + ListenerWatcher& operator=(const ListenerWatcher&) = delete; void OnListenerChanged(XdsApi::LdsUpdate listener) override { - // TODO(yashykt): Construct channel args according to received update - server_config_watcher_->UpdateConfig(nullptr); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_server_config_fetcher_trace)) { + gpr_log( + GPR_INFO, + "[ListenerWatcher %p] Received LDS update from xds client %p: %s", + this, xds_client_.get(), listener.ToString().c_str()); + } + grpc_error* error = GRPC_ERROR_NONE; + bool update_needed = UpdateXdsCertificateProvider(listener, &error); + if (error != GRPC_ERROR_NONE) { + OnError(error); + return; + } + // Only send an update, if something changed. + if (updated_once_ && !update_needed) { + return; + } + updated_once_ = true; + grpc_channel_args* updated_args = nullptr; + if (xds_certificate_provider_ != nullptr) { + grpc_arg arg_to_add = xds_certificate_provider_->MakeChannelArg(); + updated_args = grpc_channel_args_copy_and_add(args_, &arg_to_add, 1); + } else { + updated_args = grpc_channel_args_copy(args_); + } + server_config_watcher_->UpdateConfig(updated_args); } void OnError(grpc_error* error) override { @@ -97,8 +137,103 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { } private: + // Returns true if the xds certificate provider changed in a way that + // required a new security connector to be created, false otherwise. + bool UpdateXdsCertificateProvider(const XdsApi::LdsUpdate& listener, + grpc_error** error) { + // Early out if channel is not configured to use xDS security. + grpc_server_credentials* server_creds = + grpc_find_server_credentials_in_args(args_); + if (server_creds == nullptr || + server_creds->type() != kCredentialsTypeXds) { + xds_certificate_provider_ = nullptr; + return false; + } + if (xds_certificate_provider_ == nullptr) { + xds_certificate_provider_ = MakeRefCounted(); + } + // Configure root cert. + absl::string_view root_provider_instance_name = + listener.downstream_tls_context.common_tls_context + .combined_validation_context + .validation_context_certificate_provider_instance.instance_name; + absl::string_view root_provider_cert_name = + listener.downstream_tls_context.common_tls_context + .combined_validation_context + .validation_context_certificate_provider_instance + .certificate_name; + RefCountedPtr new_root_provider; + if (!root_provider_instance_name.empty()) { + new_root_provider = + xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider(root_provider_instance_name); + if (new_root_provider == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("Certificate provider instance name: \"", + root_provider_instance_name, "\" not recognized.") + .c_str()); + return false; + } + } + // Configure identity cert. + absl::string_view identity_provider_instance_name = + listener.downstream_tls_context.common_tls_context + .tls_certificate_certificate_provider_instance.instance_name; + absl::string_view identity_provider_cert_name = + listener.downstream_tls_context.common_tls_context + .tls_certificate_certificate_provider_instance.certificate_name; + RefCountedPtr new_identity_provider; + if (!identity_provider_instance_name.empty()) { + new_identity_provider = xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider( + identity_provider_instance_name); + if (new_identity_provider == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("Certificate provider instance name: \"", + identity_provider_instance_name, + "\" not recognized.") + .c_str()); + return false; + } + } + bool security_connector_update_required = false; + if (((new_root_provider == nullptr) != + (root_certificate_provider_ == nullptr)) || + ((new_identity_provider == nullptr) != + (identity_certificate_provider_ == nullptr)) || + (listener.downstream_tls_context.require_client_certificate != + xds_certificate_provider_->GetRequireClientCertificate(""))) { + security_connector_update_required = true; + } + if (root_certificate_provider_ != new_root_provider) { + root_certificate_provider_ = std::move(new_root_provider); + } + if (identity_certificate_provider_ != new_identity_provider) { + identity_certificate_provider_ = std::move(new_identity_provider); + } + xds_certificate_provider_->UpdateRootCertNameAndDistributor( + "", root_provider_cert_name, + root_certificate_provider_ == nullptr + ? nullptr + : root_certificate_provider_->distributor()); + xds_certificate_provider_->UpdateIdentityCertNameAndDistributor( + "", identity_provider_cert_name, + identity_certificate_provider_ == nullptr + ? nullptr + : identity_certificate_provider_->distributor()); + xds_certificate_provider_->UpdateRequireClientCertificate( + "", listener.downstream_tls_context.require_client_certificate); + return security_connector_update_required; + } + std::unique_ptr server_config_watcher_; + grpc_channel_args* args_; + RefCountedPtr xds_client_; + RefCountedPtr root_certificate_provider_; + RefCountedPtr identity_certificate_provider_; + RefCountedPtr xds_certificate_provider_; + bool updated_once_ = false; }; struct WatcherState { @@ -125,6 +260,7 @@ grpc_server_config_fetcher* grpc_server_config_fetcher_xds_create() { if (error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Failed to create xds client: %s", grpc_error_string(error)); + GRPC_ERROR_UNREF(error); return nullptr; } return new grpc_core::XdsServerConfigFetcher(std::move(xds_client)); diff --git a/src/core/lib/http/httpcli_security_connector.cc b/src/core/lib/http/httpcli_security_connector.cc index 15aea333c77..3ef4a4d073f 100644 --- a/src/core/lib/http/httpcli_security_connector.cc +++ b/src/core/lib/http/httpcli_security_connector.cc @@ -203,8 +203,8 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host, grpc_channel_args args = {1, &channel_arg}; c->handshake_mgr = grpc_core::MakeRefCounted(); grpc_core::HandshakerRegistry::AddHandshakers( - grpc_core::HANDSHAKER_CLIENT, &args, /*interested_parties=*/nullptr, - c->handshake_mgr.get()); + grpc_core::HANDSHAKER_CLIENT, &args, + /*interested_parties=*/nullptr, c->handshake_mgr.get()); c->handshake_mgr->DoHandshake(tcp, /*channel_args=*/nullptr, deadline, /*acceptor=*/nullptr, on_handshake_done, /*user_data=*/c); diff --git a/src/core/lib/security/credentials/alts/alts_credentials.cc b/src/core/lib/security/credentials/alts/alts_credentials.cc index 1bc76d9c0c6..30acd74948b 100644 --- a/src/core/lib/security/credentials/alts/alts_credentials.cc +++ b/src/core/lib/security/credentials/alts/alts_credentials.cc @@ -70,7 +70,8 @@ grpc_alts_server_credentials::grpc_alts_server_credentials( } grpc_core::RefCountedPtr -grpc_alts_server_credentials::create_security_connector() { +grpc_alts_server_credentials::create_security_connector( + const grpc_channel_args* /* args */) { return grpc_alts_server_security_connector_create(this->Ref()); } diff --git a/src/core/lib/security/credentials/alts/alts_credentials.h b/src/core/lib/security/credentials/alts/alts_credentials.h index cc6d5222b16..8e1362c0b61 100644 --- a/src/core/lib/security/credentials/alts/alts_credentials.h +++ b/src/core/lib/security/credentials/alts/alts_credentials.h @@ -56,7 +56,7 @@ class grpc_alts_server_credentials final : public grpc_server_credentials { ~grpc_alts_server_credentials() override; grpc_core::RefCountedPtr - create_security_connector() override; + create_security_connector(const grpc_channel_args* /* args */) override; const grpc_alts_credentials_options* options() const { return options_; } grpc_alts_credentials_options* mutable_options() { return options_; } diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 61e56e58144..cf9f402be8c 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -227,8 +227,9 @@ struct grpc_server_credentials ~grpc_server_credentials() override { DestroyProcessor(); } + // Ownership of \a args is not passed. virtual grpc_core::RefCountedPtr - create_security_connector() = 0; + create_security_connector(const grpc_channel_args* args) = 0; const char* type() const { return type_; } diff --git a/src/core/lib/security/credentials/fake/fake_credentials.cc b/src/core/lib/security/credentials/fake/fake_credentials.cc index eef636d0043..7ad4d61809a 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.cc +++ b/src/core/lib/security/credentials/fake/fake_credentials.cc @@ -59,7 +59,7 @@ class grpc_fake_server_credentials final : public grpc_server_credentials { ~grpc_fake_server_credentials() override = default; grpc_core::RefCountedPtr - create_security_connector() override { + create_security_connector(const grpc_channel_args* args) override { return grpc_fake_server_security_connector_create(this->Ref()); } }; diff --git a/src/core/lib/security/credentials/insecure/insecure_credentials.cc b/src/core/lib/security/credentials/insecure/insecure_credentials.cc index 01c1ad719da..4cf500e6f59 100644 --- a/src/core/lib/security/credentials/insecure/insecure_credentials.cc +++ b/src/core/lib/security/credentials/insecure/insecure_credentials.cc @@ -46,8 +46,8 @@ class InsecureServerCredentials final : public grpc_server_credentials { InsecureServerCredentials() : grpc_server_credentials(kCredentialsTypeInsecure) {} - RefCountedPtr create_security_connector() - override { + RefCountedPtr create_security_connector( + const grpc_channel_args* /* args */) override { return MakeRefCounted(Ref()); } }; diff --git a/src/core/lib/security/credentials/local/local_credentials.cc b/src/core/lib/security/credentials/local/local_credentials.cc index 966a887baf8..84caf1c3ba4 100644 --- a/src/core/lib/security/credentials/local/local_credentials.cc +++ b/src/core/lib/security/credentials/local/local_credentials.cc @@ -39,7 +39,8 @@ grpc_local_credentials::create_security_connector( } grpc_core::RefCountedPtr -grpc_local_server_credentials::create_security_connector() { +grpc_local_server_credentials::create_security_connector( + const grpc_channel_args* /* args */) { return grpc_local_server_security_connector_create(this->Ref()); } diff --git a/src/core/lib/security/credentials/local/local_credentials.h b/src/core/lib/security/credentials/local/local_credentials.h index 60a8a4f64ca..a1857ad8dba 100644 --- a/src/core/lib/security/credentials/local/local_credentials.h +++ b/src/core/lib/security/credentials/local/local_credentials.h @@ -50,7 +50,7 @@ class grpc_local_server_credentials final : public grpc_server_credentials { ~grpc_local_server_credentials() override = default; grpc_core::RefCountedPtr - create_security_connector() override; + create_security_connector(const grpc_channel_args* /* args */) override; grpc_local_connect_type connect_type() const { return connect_type_; } diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.cc b/src/core/lib/security/credentials/ssl/ssl_credentials.cc index e2a1ad03c0f..469eb0cfcfd 100644 --- a/src/core/lib/security/credentials/ssl/ssl_credentials.cc +++ b/src/core/lib/security/credentials/ssl/ssl_credentials.cc @@ -190,7 +190,8 @@ grpc_ssl_server_credentials::~grpc_ssl_server_credentials() { gpr_free(config_.pem_root_certs); } grpc_core::RefCountedPtr -grpc_ssl_server_credentials::create_security_connector() { +grpc_ssl_server_credentials::create_security_connector( + const grpc_channel_args* /* args */) { return grpc_ssl_server_security_connector_create(this->Ref()); } diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.h b/src/core/lib/security/credentials/ssl/ssl_credentials.h index 0c5af81c7d8..0491eea7002 100644 --- a/src/core/lib/security/credentials/ssl/ssl_credentials.h +++ b/src/core/lib/security/credentials/ssl/ssl_credentials.h @@ -69,7 +69,7 @@ class grpc_ssl_server_credentials final : public grpc_server_credentials { ~grpc_ssl_server_credentials() override; grpc_core::RefCountedPtr - create_security_connector() override; + create_security_connector(const grpc_channel_args* /* args */) override; bool has_cert_config_fetcher() const { return certificate_config_fetcher_.cb != nullptr; diff --git a/src/core/lib/security/credentials/tls/tls_credentials.cc b/src/core/lib/security/credentials/tls/tls_credentials.cc index 06887e760cc..f5b05d8a012 100644 --- a/src/core/lib/security/credentials/tls/tls_credentials.cc +++ b/src/core/lib/security/credentials/tls/tls_credentials.cc @@ -106,7 +106,8 @@ TlsServerCredentials::TlsServerCredentials( TlsServerCredentials::~TlsServerCredentials() {} grpc_core::RefCountedPtr -TlsServerCredentials::create_security_connector() { +TlsServerCredentials::create_security_connector( + const grpc_channel_args* /* args */) { return grpc_core::TlsServerSecurityConnector:: CreateTlsServerSecurityConnector(this->Ref(), options_); } diff --git a/src/core/lib/security/credentials/tls/tls_credentials.h b/src/core/lib/security/credentials/tls/tls_credentials.h index 214bc904062..a5e4f486bf9 100644 --- a/src/core/lib/security/credentials/tls/tls_credentials.h +++ b/src/core/lib/security/credentials/tls/tls_credentials.h @@ -51,7 +51,7 @@ class TlsServerCredentials final : public grpc_server_credentials { ~TlsServerCredentials() override; grpc_core::RefCountedPtr - create_security_connector() override; + create_security_connector(const grpc_channel_args* /* args */) override; grpc_tls_credentials_options* options() const { return options_.get(); } diff --git a/src/core/lib/security/credentials/xds/xds_credentials.cc b/src/core/lib/security/credentials/xds/xds_credentials.cc index cca6a71ce48..eb95090f641 100644 --- a/src/core/lib/security/credentials/xds/xds_credentials.cc +++ b/src/core/lib/security/credentials/xds/xds_credentials.cc @@ -198,9 +198,35 @@ XdsCredentials::create_security_connector( // RefCountedPtr -XdsServerCredentials::create_security_connector() { - // TODO(yashkt): Fill this - return fallback_credentials_->create_security_connector(); +XdsServerCredentials::create_security_connector(const grpc_channel_args* args) { + auto xds_certificate_provider = + XdsCertificateProvider::GetFromChannelArgs(args); + // Identity certs are a must for TLS. + if (xds_certificate_provider != nullptr && + xds_certificate_provider->ProvidesIdentityCerts("")) { + auto tls_credentials_options = + MakeRefCounted(); + tls_credentials_options->set_watch_identity_pair(true); + tls_credentials_options->set_certificate_provider(xds_certificate_provider); + if (xds_certificate_provider->ProvidesRootCerts("")) { + tls_credentials_options->set_watch_root_cert(true); + if (xds_certificate_provider->GetRequireClientCertificate("")) { + tls_credentials_options->set_cert_request_type( + GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); + } else { + tls_credentials_options->set_cert_request_type( + GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); + } + } else { + // Do not request client certificate if there is no way to verify. + tls_credentials_options->set_cert_request_type( + GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); + } + auto tls_credentials = MakeRefCounted( + std::move(tls_credentials_options)); + return tls_credentials->create_security_connector(args); + } + return fallback_credentials_->create_security_connector(args); } } // namespace grpc_core diff --git a/src/core/lib/security/credentials/xds/xds_credentials.h b/src/core/lib/security/credentials/xds/xds_credentials.h index e12608c9d40..f362126e285 100644 --- a/src/core/lib/security/credentials/xds/xds_credentials.h +++ b/src/core/lib/security/credentials/xds/xds_credentials.h @@ -52,8 +52,8 @@ class XdsServerCredentials final : public grpc_server_credentials { : grpc_server_credentials(kCredentialsTypeXds), fallback_credentials_(std::move(fallback_credentials)) {} - RefCountedPtr create_security_connector() - override; + RefCountedPtr create_security_connector( + const grpc_channel_args* /* args */) override; private: RefCountedPtr fallback_credentials_; diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 06bdd4d7a9d..1fbbd820371 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -415,12 +415,15 @@ struct grpc_server_config_fetcher { class WatcherInterface { public: virtual ~WatcherInterface() = default; + // Ownership of \a args is transferred. virtual void UpdateConfig(grpc_channel_args* args) = 0; }; virtual ~grpc_server_config_fetcher() = default; + // Ownership of \a args is transferred. virtual void StartWatch(std::string listening_address, + grpc_channel_args* args, std::unique_ptr watcher) = 0; virtual void CancelWatch(WatcherInterface* watcher) = 0; virtual grpc_pollset_set* interested_parties() = 0; diff --git a/src/proto/grpc/testing/xds/v3/BUILD b/src/proto/grpc/testing/xds/v3/BUILD index 1bcfa4b510e..ce99015c70c 100644 --- a/src/proto/grpc/testing/xds/v3/BUILD +++ b/src/proto/grpc/testing/xds/v3/BUILD @@ -107,6 +107,10 @@ grpc_proto_library( "listener.proto", ], well_known_protos = True, + deps = [ + "address_proto", + "base_proto", + ], ) grpc_proto_library( diff --git a/src/proto/grpc/testing/xds/v3/listener.proto b/src/proto/grpc/testing/xds/v3/listener.proto index e9160e033e2..40ce59cbe09 100644 --- a/src/proto/grpc/testing/xds/v3/listener.proto +++ b/src/proto/grpc/testing/xds/v3/listener.proto @@ -18,6 +18,9 @@ syntax = "proto3"; package envoy.config.listener.v3; +import "src/proto/grpc/testing/xds/v3/address.proto"; +import "src/proto/grpc/testing/xds/v3/base.proto"; + import "google/protobuf/any.proto"; // [#protodoc-title: Listener configuration] @@ -37,6 +40,44 @@ message ApiListener { google.protobuf.Any api_listener = 1; } +message FilterChainMatch { + // If non-empty, a list of application protocols (e.g. ALPN for TLS protocol) to consider when + // determining a filter chain match. Those values will be compared against the application + // protocols of a new connection, when detected by one of the listener filters. + // + // Suggested values include: + // + // * ``http/1.1`` - set by :ref:`envoy.filters.listener.tls_inspector + // `, + // * ``h2`` - set by :ref:`envoy.filters.listener.tls_inspector ` + // + // .. attention:: + // + // Currently, only :ref:`TLS Inspector ` provides + // application protocol detection based on the requested + // `ALPN `_ values. + // + // However, the use of ALPN is pretty much limited to the HTTP/2 traffic on the Internet, + // and matching on values other than ``h2`` is going to lead to a lot of false negatives, + // unless all connecting clients are known to use ALPN. + repeated string application_protocols = 10; +} + +// A filter chain wraps a set of match criteria, an option TLS context, a set of filters, and +// various other parameters. +// [#next-free-field: 10] +message FilterChain { + // The criteria to use when matching a connection to this filter chain. + FilterChainMatch filter_chain_match = 1; + + // Optional custom transport socket implementation to use for downstream connections. + // To setup TLS, set a transport socket with name `tls` and + // :ref:`DownstreamTlsContext ` in the `typed_config`. + // If no transport socket configuration is specified, new connections + // will be set up with plaintext. + core.v3.TransportSocket transport_socket = 6; +} + // [#next-free-field: 23] message Listener { // The unique name by which this listener is known. If no name is provided, @@ -44,6 +85,20 @@ message Listener { // updated or removed via :ref:`LDS ` a unique name must be provided. string name = 1; + // The address that the listener should listen on. In general, the address must be unique, though + // that is governed by the bind rules of the OS. E.g., multiple listeners can listen on port 0 on + // Linux as the actual port will be allocated by the OS. + core.v3.Address address = 2; + + // A list of filter chains to consider for this listener. The + // :ref:`FilterChain ` with the most specific + // :ref:`FilterChainMatch ` criteria is used on a + // connection. + // + // Example using SNI for filter chain selection can be found in the + // :ref:`FAQ entry `. + repeated FilterChain filter_chains = 3; + // Used to represent an API listener, which is used in non-proxy clients. The type of API // exposed to the non-proxy application depends on the type of API listener. // When this field is set, no other field except for :ref:`name` diff --git a/src/proto/grpc/testing/xds/v3/tls.proto b/src/proto/grpc/testing/xds/v3/tls.proto index 47db0a0b9c6..e9acfb52a62 100644 --- a/src/proto/grpc/testing/xds/v3/tls.proto +++ b/src/proto/grpc/testing/xds/v3/tls.proto @@ -20,6 +20,8 @@ package envoy.extensions.transport_sockets.tls.v3; import "src/proto/grpc/testing/xds/v3/string.proto"; +import "google/protobuf/wrappers.proto"; + message CertificateValidationContext { // An optional list of Subject Alternative name matchers. If specified, Envoy will verify that the // Subject Alternative Name of the presented certificate matches one of the specified matchers. @@ -53,6 +55,16 @@ message UpstreamTlsContext { CommonTlsContext common_tls_context = 1; } +message DownstreamTlsContext { + // Common TLS context settings. + CommonTlsContext common_tls_context = 1; + + // If specified, Envoy will reject connections without a valid client + // certificate. + google.protobuf.BoolValue require_client_certificate = 2; +} + + // TLS context shared by both client and server TLS contexts. // [#next-free-field: 14] message CommonTlsContext { diff --git a/test/core/end2end/h2_ssl_session_reuse_test.cc b/test/core/end2end/h2_ssl_session_reuse_test.cc index 4aeac24395a..9230a56fabd 100644 --- a/test/core/end2end/h2_ssl_session_reuse_test.cc +++ b/test/core/end2end/h2_ssl_session_reuse_test.cc @@ -193,7 +193,6 @@ void do_round_trip(grpc_completion_queue* cq, grpc_server* server, auth, GRPC_SSL_SESSION_REUSED_PROPERTY); const grpc_auth_property* property = grpc_auth_property_iterator_next(&it); GPR_ASSERT(property != nullptr); - if (expect_session_reuse) { GPR_ASSERT(strcmp(property->value, "true") == 0); } else { diff --git a/test/core/security/grpc_tls_credentials_options_test.cc b/test/core/security/grpc_tls_credentials_options_test.cc index 148e4a07522..c5736ca1d77 100644 --- a/test/core/security/grpc_tls_credentials_options_test.cc +++ b/test/core/security/grpc_tls_credentials_options_test.cc @@ -182,7 +182,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -201,7 +201,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, options->set_cert_request_type(GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -220,7 +220,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, options->set_cert_request_type(GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -325,7 +325,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -344,7 +344,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, options->set_cert_request_type(GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -363,7 +363,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, options->set_cert_request_type(GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -380,7 +380,7 @@ TEST_F(GrpcTlsCredentialsOptionsTest, options->set_cert_request_type(GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE); auto credentials = MakeRefCounted(options); ASSERT_NE(credentials, nullptr); - auto connector = credentials->create_security_connector(); + auto connector = credentials->create_security_connector(nullptr); ASSERT_NE(connector, nullptr); TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); diff --git a/test/core/security/ssl_server_fuzzer.cc b/test/core/security/ssl_server_fuzzer.cc index bb5ca7227c0..cdd00e0633a 100644 --- a/test/core/security/ssl_server_fuzzer.cc +++ b/test/core/security/ssl_server_fuzzer.cc @@ -91,7 +91,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // Create security connector grpc_core::RefCountedPtr sc = - creds->create_security_connector(); + creds->create_security_connector(nullptr); GPR_ASSERT(sc != nullptr); grpc_millis deadline = GPR_MS_PER_SEC + grpc_core::ExecCtx::Get()->Now(); diff --git a/test/core/security/tls_security_connector_test.cc b/test/core/security/tls_security_connector_test.cc index aff048f48f9..5c94e89c1d2 100644 --- a/test/core/security/tls_security_connector_test.cc +++ b/test/core/security/tls_security_connector_test.cc @@ -392,7 +392,7 @@ TEST_F(TlsSecurityConnectorTest, grpc_core::RefCountedPtr credential = grpc_core::MakeRefCounted(options); grpc_core::RefCountedPtr connector = - credential->create_security_connector(); + credential->create_security_connector(nullptr); EXPECT_NE(connector, nullptr); grpc_core::TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -429,7 +429,7 @@ TEST_F(TlsSecurityConnectorTest, grpc_core::RefCountedPtr identity_credential = grpc_core::MakeRefCounted(identity_options); grpc_core::RefCountedPtr identity_connector = - identity_credential->create_security_connector(); + identity_credential->create_security_connector(nullptr); EXPECT_NE(identity_connector, nullptr); grpc_core::TlsServerSecurityConnector* tls_identity_connector = static_cast( @@ -466,7 +466,7 @@ TEST_F(TlsSecurityConnectorTest, grpc_core::RefCountedPtr credential = grpc_core::MakeRefCounted(options); grpc_core::RefCountedPtr connector = - credential->create_security_connector(); + credential->create_security_connector(nullptr); EXPECT_NE(connector, nullptr); grpc_core::TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); @@ -500,7 +500,7 @@ TEST_F(TlsSecurityConnectorTest, grpc_core::RefCountedPtr credential = grpc_core::MakeRefCounted(options); grpc_core::RefCountedPtr connector = - credential->create_security_connector(); + credential->create_security_connector(nullptr); EXPECT_NE(connector, nullptr); grpc_core::TlsServerSecurityConnector* tls_connector = static_cast(connector.get()); diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 323f42101e2..9b719e89c05 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -29,8 +29,10 @@ #include #include +#include "absl/functional/bind_front.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "absl/types/optional.h" #include @@ -44,6 +46,7 @@ #include #include #include +#include #include "src/core/ext/filters/client_channel/backup_poller.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" @@ -102,6 +105,7 @@ using ::envoy::config::listener::v3::Listener; using ::envoy::config::route::v3::RouteConfiguration; using ::envoy::extensions::filters::network::http_connection_manager::v3:: HttpConnectionManager; +using ::envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext; using ::envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext; using ::envoy::type::matcher::v3::StringMatcher; using ::envoy::type::v3::FractionalPercent; @@ -1501,24 +1505,13 @@ std::shared_ptr CreateTlsFallbackCredentials() { class XdsEnd2endTest : public ::testing::TestWithParam { protected: XdsEnd2endTest(size_t num_backends, size_t num_balancers, - int client_load_reporting_interval_seconds = 100) + int client_load_reporting_interval_seconds = 100, + bool use_xds_enabled_server = false) : num_backends_(num_backends), num_balancers_(num_balancers), client_load_reporting_interval_seconds_( - client_load_reporting_interval_seconds) {} - - static void SetUpTestCase() { - // Make the backup poller poll very frequently in order to pick up - // updates from all the subchannels's FDs. - GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); -#if TARGET_OS_IPHONE - // Workaround Apple CFStream bug - gpr_setenv("grpc_cfstream", "0"); -#endif - grpc_init(); - } - - static void TearDownTestCase() { grpc_shutdown(); } + client_load_reporting_interval_seconds), + use_xds_enabled_server_(use_xds_enabled_server) {} void SetUp() override { gpr_setenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT", "true"); @@ -1576,7 +1569,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam { } // Start the backends. for (size_t i = 0; i < num_backends_; ++i) { - backends_.emplace_back(new BackendServerThread); + backends_.emplace_back(new BackendServerThread(use_xds_enabled_server_)); backends_.back()->Start(); } // Start the load balancers. @@ -2053,7 +2046,9 @@ class XdsEnd2endTest : public ::testing::TestWithParam { protected: class ServerThread { public: - ServerThread() : port_(g_port_saver->GetPort()) {} + explicit ServerThread(bool use_xds_enabled_server = false) + : port_(g_port_saver->GetPort()), + use_xds_enabled_server_(use_xds_enabled_server) {} virtual ~ServerThread(){}; void Start() { @@ -2078,10 +2073,17 @@ class XdsEnd2endTest : public ::testing::TestWithParam { grpc_core::MutexLock lock(mu); std::ostringstream server_address; server_address << "localhost:" << port_; - ServerBuilder builder; - builder.AddListeningPort(server_address.str(), Credentials()); - RegisterAllServices(&builder); - server_ = builder.BuildAndStart(); + if (use_xds_enabled_server_) { + experimental::XdsServerBuilder builder; + builder.AddListeningPort(server_address.str(), Credentials()); + RegisterAllServices(&builder); + server_ = builder.BuildAndStart(); + } else { + ServerBuilder builder; + builder.AddListeningPort(server_address.str(), Credentials()); + RegisterAllServices(&builder); + server_ = builder.BuildAndStart(); + } cond->Signal(); } @@ -2102,6 +2104,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { int port() const { return port_; } + bool use_xds_enabled_server() const { return use_xds_enabled_server_; } + private: virtual void RegisterAllServices(ServerBuilder* builder) = 0; virtual void StartAllServices() = 0; @@ -2113,10 +2117,14 @@ class XdsEnd2endTest : public ::testing::TestWithParam { std::unique_ptr server_; std::unique_ptr thread_; bool running_ = false; + const bool use_xds_enabled_server_; }; class BackendServerThread : public ServerThread { public: + explicit BackendServerThread(bool use_xds_enabled_server) + : ServerThread(use_xds_enabled_server) {} + BackendServiceImpl<::grpc::testing::EchoTestService::Service>* backend_service() { return &backend_service_; @@ -2132,21 +2140,28 @@ class XdsEnd2endTest : public ::testing::TestWithParam { std::shared_ptr Credentials() override { if (GetParam().use_xds_credentials()) { - std::string root_cert = ReadFile(kCaCertPath); - std::string identity_cert = ReadFile(kServerCertPath); - std::string private_key = ReadFile(kServerKeyPath); - std::vector identity_key_cert_pairs = - {{private_key, identity_cert}}; - auto certificate_provider = - std::make_shared( - root_cert, identity_key_cert_pairs); - grpc::experimental::TlsServerCredentialsOptions options( - certificate_provider); - options.watch_root_certs(); - options.watch_identity_key_cert_pairs(); - options.set_cert_request_type( - GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); - return grpc::experimental::TlsServerCredentials(options); + if (use_xds_enabled_server()) { + // We are testing server's use of XdsServerCredentials + return experimental::XdsServerCredentials( + InsecureServerCredentials()); + } else { + // We are testing client's use of XdsCredentials + std::string root_cert = ReadFile(kCaCertPath); + std::string identity_cert = ReadFile(kServerCertPath); + std::string private_key = ReadFile(kServerKeyPath); + std::vector + identity_key_cert_pairs = {{private_key, identity_cert}}; + auto certificate_provider = std::make_shared< + grpc::experimental::StaticDataCertificateProvider>( + root_cert, identity_key_cert_pairs); + grpc::experimental::TlsServerCredentialsOptions options( + certificate_provider); + options.watch_root_certs(); + options.watch_identity_key_cert_pairs(); + options.set_cert_request_type( + GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); + return grpc::experimental::TlsServerCredentials(options); + } } return ServerThread::Credentials(); } @@ -2256,6 +2271,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam { Listener default_listener_; RouteConfiguration default_route_config_; Cluster default_cluster_; + bool use_xds_enabled_server_; }; class BasicTest : public XdsEnd2endTest { @@ -2952,7 +2968,8 @@ TEST_P(LdsTest, NoApiListener) { const auto& response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); - EXPECT_EQ(response_state.error_message, "Listener has no ApiListener."); + EXPECT_EQ(response_state.error_message, + "Listener has neither address nor ApiListener"); } // Tests that LDS client should send a NACK if the route_specifier in the @@ -5334,12 +5351,6 @@ class XdsSecurityTest : public BasicTest { static void SetUpTestCase() { gpr_setenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT", "true"); BasicTest::SetUpTestCase(); - grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory( - absl::make_unique( - "fake1", &g_fake1_cert_data_map)); - grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory( - absl::make_unique( - "fake2", &g_fake2_cert_data_map)); } static void TearDownTestCase() { @@ -5442,16 +5453,33 @@ class XdsSecurityTest : public BasicTest { StartBackend(0); ResetBackendCounters(); if (test_expects_failure) { - if (!SendRpc().ok()) break; + Status status = SendRpc(); + if (status.ok()) { + gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again."); + continue; + } } else { WaitForBackend(0); - if (SendRpc().ok() && - backends_[0]->backend_service()->request_count() == 1UL && - backends_[0]->backend_service()->last_peer_identity() == - expected_authenticated_identity) { - break; + Status status = SendRpc(); + if (!status.ok()) { + gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.", + status.error_code(), status.error_message().c_str()); + continue; + } + if (backends_[0]->backend_service()->last_peer_identity() != + expected_authenticated_identity) { + gpr_log( + GPR_ERROR, + "Expected client identity does not match. (actual) %s vs " + "(expected) %s Trying again.", + absl::StrJoin( + backends_[0]->backend_service()->last_peer_identity(), ",") + .c_str(), + absl::StrJoin(expected_authenticated_identity, ",").c_str()); + continue; } } + break; } EXPECT_LT(num_tries, kRetryCount); } @@ -5935,6 +5963,597 @@ TEST_P(XdsSecurityTest, TestFileWatcherCertificateProvider) { authenticated_identity_); } +class XdsEnabledServerTest : public XdsEnd2endTest { + protected: + XdsEnabledServerTest() + : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {} + + void SetUp() override { + XdsEnd2endTest::SetUp(); + AdsServiceImpl::EdsResourceArgs args({ + {"locality0", GetBackendPorts(0, 1)}, + }); + balancers_[0]->ads_service()->SetEdsResource( + BuildEdsResource(args, DefaultEdsServiceName())); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + } +}; + +TEST_P(XdsEnabledServerTest, Basic) { + Listener listener; + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=127.0.0.1:", + backends_[0]->port())); + listener.mutable_address()->mutable_socket_address()->set_address( + "127.0.0.1"); + listener.mutable_address()->mutable_socket_address()->set_port_value( + backends_[0]->port()); + listener.add_filter_chains(); + balancers_[0]->ads_service()->SetLdsResource(listener); + WaitForBackend(0); + CheckRpcSendOk(); +} + +TEST_P(XdsEnabledServerTest, BadLdsUpdateNoApiListenerNorAddress) { + Listener listener; + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=127.0.0.1:", + backends_[0]->port())); + listener.add_filter_chains(); + balancers_[0]->ads_service()->SetLdsResource(listener); + // TODO(yashykt): We need to set responses for both addresses because of + // b/176843510 + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", + backends_[0]->port())); + balancers_[0]->ads_service()->SetLdsResource(listener); + CheckRpcSendFailure(1, RpcOptions().set_wait_for_ready(true)); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Listener has neither address nor ApiListener"); +} + +TEST_P(XdsEnabledServerTest, BadLdsUpdateBothApiListenerAndAddress) { + Listener listener; + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=127.0.0.1:", + backends_[0]->port())); + balancers_[0]->ads_service()->SetLdsResource(listener); + listener.mutable_address()->mutable_socket_address()->set_address( + "127.0.0.1"); + listener.mutable_address()->mutable_socket_address()->set_port_value( + backends_[0]->port()); + auto* filter_chain = listener.add_filter_chains(); + auto* transport_socket = filter_chain->mutable_transport_socket(); + transport_socket->set_name("envoy.transport_sockets.tls"); + listener.mutable_api_listener(); + balancers_[0]->ads_service()->SetLdsResource(listener); + // TODO(yashykt): We need to set responses for both addresses because of + // b/176843510 + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", + backends_[0]->port())); + balancers_[0]->ads_service()->SetLdsResource(listener); + CheckRpcSendFailure(1, RpcOptions().set_wait_for_ready(true)); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "Listener has both address and ApiListener"); +} + +class XdsServerSecurityTest : public XdsEnd2endTest { + protected: + XdsServerSecurityTest() + : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {} + + static void SetUpTestCase() { + gpr_setenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT", "true"); + XdsEnd2endTest::SetUpTestCase(); + } + + static void TearDownTestCase() { + XdsEnd2endTest::TearDownTestCase(); + gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT"); + } + + void SetUp() override { + XdsEnd2endTest::SetUp(); + root_cert_ = ReadFile(kCaCertPath); + bad_root_cert_ = ReadFile(kBadClientCertPath); + identity_pair_ = ReadTlsIdentityPair(kServerKeyPath, kServerCertPath); + bad_identity_pair_ = + ReadTlsIdentityPair(kBadClientKeyPath, kBadClientCertPath); + identity_pair_2_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath); + server_authenticated_identity_ = {"*.test.google.fr", + "waterzooi.test.google.be", + "*.test.youtube.com", "192.168.1.3"}; + server_authenticated_identity_2_ = {"testclient"}; + client_authenticated_identity_ = {"*.test.google.fr", + "waterzooi.test.google.be", + "*.test.youtube.com", "192.168.1.3"}; + AdsServiceImpl::EdsResourceArgs args({ + {"locality0", GetBackendPorts(0, 1)}, + }); + balancers_[0]->ads_service()->SetEdsResource( + BuildEdsResource(args, DefaultEdsServiceName())); + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + } + + void TearDown() override { + g_fake1_cert_data_map = nullptr; + g_fake2_cert_data_map = nullptr; + XdsEnd2endTest::TearDown(); + } + + void SetLdsUpdate(absl::string_view root_instance_name, + absl::string_view root_certificate_name, + absl::string_view identity_instance_name, + absl::string_view identity_certificate_name, + bool require_client_certificates) { + Listener listener; + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=127.0.0.1:", + backends_[0]->port())); + listener.mutable_address()->mutable_socket_address()->set_address( + "127.0.0.1"); + listener.mutable_address()->mutable_socket_address()->set_port_value( + backends_[0]->port()); + auto* filter_chain = listener.add_filter_chains(); + if (!identity_instance_name.empty()) { + auto* transport_socket = filter_chain->mutable_transport_socket(); + transport_socket->set_name("envoy.transport_sockets.tls"); + DownstreamTlsContext downstream_tls_context; + downstream_tls_context.mutable_common_tls_context() + ->mutable_tls_certificate_certificate_provider_instance() + ->set_instance_name(std::string(identity_instance_name)); + downstream_tls_context.mutable_common_tls_context() + ->mutable_tls_certificate_certificate_provider_instance() + ->set_certificate_name(std::string(identity_certificate_name)); + if (!root_instance_name.empty()) { + downstream_tls_context.mutable_common_tls_context() + ->mutable_combined_validation_context() + ->mutable_validation_context_certificate_provider_instance() + ->set_instance_name(std::string(root_instance_name)); + downstream_tls_context.mutable_common_tls_context() + ->mutable_combined_validation_context() + ->mutable_validation_context_certificate_provider_instance() + ->set_certificate_name(std::string(root_certificate_name)); + downstream_tls_context.mutable_require_client_certificate()->set_value( + require_client_certificates); + } + transport_socket->mutable_typed_config()->PackFrom( + downstream_tls_context); + } + balancers_[0]->ads_service()->SetLdsResource(listener); + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", + backends_[0]->port())); + listener.mutable_address()->mutable_socket_address()->set_address("[::1]"); + balancers_[0]->ads_service()->SetLdsResource(listener); + } + + std::shared_ptr CreateMtlsChannel() { + ChannelArguments args; + // Override target name for host name check + args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, + ipv6_only_ ? "[::1]" : "127.0.0.1"); + args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + std::string uri = absl::StrCat( + ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port()); + // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed. + grpc_tls_credentials_options* options = + grpc_tls_credentials_options_create(); + grpc_tls_credentials_options_set_server_verification_option( + options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION); + grpc_tls_credentials_options_set_certificate_provider( + options, + grpc_core::MakeRefCounted( + ReadFile(kCaCertPath), + ReadTlsIdentityPair(kServerKeyPath, kServerCertPath)) + .get()); + grpc_tls_credentials_options_watch_root_certs(options); + grpc_tls_credentials_options_watch_identity_key_cert_pairs(options); + grpc_tls_server_authorization_check_config* check_config = + grpc_tls_server_authorization_check_config_create( + nullptr, ServerAuthCheckSchedule, nullptr, nullptr); + grpc_tls_credentials_options_set_server_authorization_check_config( + options, check_config); + auto channel_creds = std::make_shared( + grpc_tls_credentials_create(options)); + grpc_tls_server_authorization_check_config_release(check_config); + return CreateCustomChannel(uri, channel_creds, args); + } + + std::shared_ptr CreateTlsChannel() { + ChannelArguments args; + // Override target name for host name check + args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, + ipv6_only_ ? "[::1]" : "127.0.0.1"); + args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + std::string uri = absl::StrCat( + ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port()); + // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed. + grpc_tls_credentials_options* options = + grpc_tls_credentials_options_create(); + grpc_tls_credentials_options_set_server_verification_option( + options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION); + grpc_tls_credentials_options_set_certificate_provider( + options, + grpc_core::MakeRefCounted( + ReadFile(kCaCertPath), + ReadTlsIdentityPair(kServerKeyPath, kServerCertPath)) + .get()); + grpc_tls_credentials_options_watch_root_certs(options); + grpc_tls_server_authorization_check_config* check_config = + grpc_tls_server_authorization_check_config_create( + nullptr, ServerAuthCheckSchedule, nullptr, nullptr); + grpc_tls_credentials_options_set_server_authorization_check_config( + options, check_config); + auto channel_creds = std::make_shared( + grpc_tls_credentials_create(options)); + grpc_tls_server_authorization_check_config_release(check_config); + return CreateCustomChannel(uri, channel_creds, args); + } + + std::shared_ptr CreateInsecureChannel() { + ChannelArguments args; + // Override target name for host name check + args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, + ipv6_only_ ? "[::1]" : "127.0.0.1"); + args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + std::string uri = absl::StrCat( + ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port()); + return CreateCustomChannel(uri, InsecureChannelCredentials(), args); + } + + void SendRpc(std::function()> channel_creator, + std::vector expected_server_identity, + std::vector expected_client_identity, + bool test_expects_failure = false) { + gpr_log(GPR_INFO, "Sending RPC"); + int num_tries = 0; + constexpr int kRetryCount = 10; + for (; num_tries < kRetryCount; num_tries++) { + auto channel = channel_creator(); + auto stub = grpc::testing::EchoTestService::NewStub(channel); + ClientContext context; + context.set_wait_for_ready(true); + context.set_deadline(grpc_timeout_milliseconds_to_deadline(2000)); + EchoRequest request; + request.set_message(kRequestMessage); + EchoResponse response; + Status status = stub->Echo(&context, request, &response); + if (test_expects_failure) { + if (status.ok()) { + gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again."); + continue; + } + } else { + if (!status.ok()) { + gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.", + status.error_code(), status.error_message().c_str()); + continue; + } + EXPECT_EQ(response.message(), kRequestMessage); + std::vector peer_identity; + for (const auto& entry : context.auth_context()->GetPeerIdentity()) { + peer_identity.emplace_back( + std::string(entry.data(), entry.size()).c_str()); + } + if (peer_identity != expected_server_identity) { + gpr_log(GPR_ERROR, + "Expected server identity does not match. (actual) %s vs " + "(expected) %s Trying again.", + absl::StrJoin(peer_identity, ",").c_str(), + absl::StrJoin(expected_server_identity, ",").c_str()); + continue; + } + if (backends_[0]->backend_service()->last_peer_identity() != + expected_client_identity) { + gpr_log( + GPR_ERROR, + "Expected client identity does not match. (actual) %s vs " + "(expected) %s Trying again.", + absl::StrJoin( + backends_[0]->backend_service()->last_peer_identity(), ",") + .c_str(), + absl::StrJoin(expected_client_identity, ",").c_str()); + continue; + } + } + break; + } + EXPECT_LT(num_tries, kRetryCount); + } + + std::string root_cert_; + std::string bad_root_cert_; + grpc_core::PemKeyCertPairList identity_pair_; + grpc_core::PemKeyCertPairList bad_identity_pair_; + grpc_core::PemKeyCertPairList identity_pair_2_; + std::vector server_authenticated_identity_; + std::vector server_authenticated_identity_2_; + std::vector client_authenticated_identity_; +}; + +TEST_P(XdsServerSecurityTest, TlsConfigurationWithoutRootProviderInstance) { + Listener listener; + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=127.0.0.1:", + backends_[0]->port())); + balancers_[0]->ads_service()->SetLdsResource(listener); + auto* socket_address = listener.mutable_address()->mutable_socket_address(); + socket_address->set_address("127.0.0.1"); + socket_address->set_port_value(backends_[0]->port()); + auto* filter_chain = listener.add_filter_chains(); + auto* transport_socket = filter_chain->mutable_transport_socket(); + transport_socket->set_name("envoy.transport_sockets.tls"); + DownstreamTlsContext downstream_tls_context; + transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context); + balancers_[0]->ads_service()->SetLdsResource(listener); + // TODO(yashykt): We need to set responses for both addresses because of + // b/176843510. + listener.set_name( + absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", + backends_[0]->port())); + socket_address->set_address("[::1]"); + balancers_[0]->ads_service()->SetLdsResource(listener); + CheckRpcSendFailure(1, RpcOptions().set_wait_for_ready(true)); + const auto& response_state = + balancers_[0]->ads_service()->lds_response_state(); + EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); + EXPECT_EQ(response_state.error_message, + "TLS configuration provided but no " + "tls_certificate_certificate_provider_instance found."); +} + +TEST_P(XdsServerSecurityTest, UnknownIdentityCertificateProvider) { + SetLdsUpdate("", "", "unknown", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, + true /* test_expects_failure */); +} + +TEST_P(XdsServerSecurityTest, UnknownRootCertificateProvider) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + SetLdsUpdate("unknown", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, + true /* test_expects_failure */); +} + +TEST_P(XdsServerSecurityTest, TestMtls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithRootPluginUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + FakeCertificateProvider::CertDataMap fake2_cert_map = { + {"", {bad_root_cert_, bad_identity_pair_}}}; + g_fake2_cert_data_map = &fake2_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin2", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, + true /* test_expects_failure */); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityPluginUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + FakeCertificateProvider::CertDataMap fake2_cert_map = { + {"", {root_cert_, identity_pair_2_}}}; + g_fake2_cert_data_map = &fake2_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin1", "", "fake_plugin2", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_2_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithBothPluginsUpdated) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + FakeCertificateProvider::CertDataMap fake2_cert_map = { + {"good", {root_cert_, identity_pair_2_}}, + {"", {bad_root_cert_, bad_identity_pair_}}}; + g_fake2_cert_data_map = &fake2_cert_map; + SetLdsUpdate("fake_plugin2", "", "fake_plugin2", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, + true /* test_expects_failure */); + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin2", "good", "fake_plugin2", "good", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_2_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithRootCertificateNameUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}, + {"bad", {bad_root_cert_, bad_identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin1", "bad", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, + true /* test_expects_failure */); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityCertificateNameUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}, + {"good", {root_cert_, identity_pair_2_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "good", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_2_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsWithBothCertificateNamesUpdated) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}, + {"good", {root_cert_, identity_pair_2_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("fake_plugin1", "good", "fake_plugin1", "good", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_2_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringButProvidingClientCerts) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringAndNotProvidingClientCerts) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); +} + +TEST_P(XdsServerSecurityTest, TestTls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); +} + +TEST_P(XdsServerSecurityTest, TestTlsWithIdentityPluginUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + FakeCertificateProvider::CertDataMap fake2_cert_map = { + {"", {root_cert_, identity_pair_2_}}}; + g_fake2_cert_data_map = &fake2_cert_map; + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); + SetLdsUpdate("", "", "fake_plugin2", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_2_, {}); +} + +TEST_P(XdsServerSecurityTest, TestTlsWithIdentityCertificateNameUpdate) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}, + {"good", {root_cert_, identity_pair_2_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); + SetLdsUpdate("", "", "fake_plugin1", "good", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_2_, {}); +} + +TEST_P(XdsServerSecurityTest, TestFallback) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "", "", false); + SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); +} + +TEST_P(XdsServerSecurityTest, TestMtlsToTls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, + true /* test_expects_failure */); + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); +} + +TEST_P(XdsServerSecurityTest, TestTlsToMtls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, + true /* test_expects_failure */); +} + +TEST_P(XdsServerSecurityTest, TestMtlsToFallback) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); + SetLdsUpdate("", "", "", "", false); + SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); +} + +TEST_P(XdsServerSecurityTest, TestFallbackToMtls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "", "", false); + SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); + SendRpc([this]() { return CreateMtlsChannel(); }, + server_authenticated_identity_, client_authenticated_identity_); +} + +TEST_P(XdsServerSecurityTest, TestTlsToFallback) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); + SetLdsUpdate("", "", "", "", false); + SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); +} + +TEST_P(XdsServerSecurityTest, TestFallbackToTls) { + FakeCertificateProvider::CertDataMap fake1_cert_map = { + {"", {root_cert_, identity_pair_}}}; + g_fake1_cert_data_map = &fake1_cert_map; + SetLdsUpdate("", "", "", "", false); + SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SetLdsUpdate("", "", "fake_plugin1", "", false); + SendRpc([this]() { return CreateTlsChannel(); }, + server_authenticated_identity_, {}); +} + using EdsTest = BasicTest; // Tests that EDS client should send a NACK if the EDS update contains @@ -7266,6 +7885,18 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, XdsSecurityTest, true)), &TestTypeName); +// We are only testing the server here. +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerTest, + ::testing::Values(TestType(true, false, false, false, + false)), + &TestTypeName); + +// We are only testing the server here. +INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerSecurityTest, + ::testing::Values(TestType(false, false, false, false, + true)), + &TestTypeName); + // EDS could be tested with or without XdsResolver, but the tests would // be the same either way, so we test it only with XdsResolver. INSTANTIATE_TEST_SUITE_P(XdsTest, EdsTest, @@ -7340,6 +7971,21 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); grpc::testing::WriteBootstrapFiles(); grpc::testing::g_port_saver = new grpc::testing::PortSaver(); + // Make the backup poller poll very frequently in order to pick up + // updates from all the subchannels's FDs. + GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); +#if TARGET_OS_IPHONE + // Workaround Apple CFStream bug + gpr_setenv("grpc_cfstream", "0"); +#endif + grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory( + absl::make_unique( + "fake1", &grpc::testing::g_fake1_cert_data_map)); + grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory( + absl::make_unique( + "fake2", &grpc::testing::g_fake2_cert_data_map)); + grpc_init(); const auto result = RUN_ALL_TESTS(); + grpc_shutdown(); return result; }