From c7101d086736a32d992fd2df803b6116f7c186fc Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 2 Jan 2024 09:17:12 -0800 Subject: [PATCH] [xDS] move CDS and EDS watchers into xds resolver (#35011) Implements gRFC A74 (https://github.com/grpc/proposal/pull/404). Closes #35011 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35011 from markdroth:xds_watchers_in_xds_resolver a39f71f37fc188f26dafc8fa8d09251e08abbc07 PiperOrigin-RevId: 595134549 --- BUILD | 1 - CMakeLists.txt | 3 +- Makefile | 6 +- Package.swift | 7 +- build_autogenerated.yaml | 7 +- config.m4 | 3 +- config.w32 | 3 +- doc/environment_variables.md | 1 - gRPC-C++.podspec | 8 +- gRPC-Core.podspec | 11 +- grpc.gemspec | 7 +- grpc.gyp | 3 +- package.xml | 7 +- src/core/BUILD | 109 +- .../client_channel/lb_policy/xds/cds.cc | 1037 +++++++------- .../lb_policy/xds/xds_cluster_impl.cc | 334 +++-- .../lb_policy/xds/xds_cluster_manager.cc | 2 +- .../lb_policy/xds/xds_cluster_resolver.cc | 1231 ----------------- .../lb_policy/xds/xds_override_host.cc | 121 +- .../lb_policy/xds/xds_override_host.h | 8 +- .../resolver/xds/xds_dependency_manager.cc | 1039 ++++++++++++++ .../resolver/xds/xds_dependency_manager.h | 281 ++++ .../resolver/xds/xds_resolver.cc | 365 ++--- ...s_resolver.h => xds_resolver_attributes.h} | 9 +- .../resolver/xds/xds_resolver_trace.cc | 25 + .../resolver/xds/xds_resolver_trace.h | 30 + .../stateful_session_filter.cc | 2 +- src/core/ext/xds/xds_bootstrap.h | 3 + src/core/ext/xds/xds_certificate_provider.cc | 336 ++--- src/core/ext/xds/xds_certificate_provider.h | 132 +- src/core/ext/xds/xds_cluster.cc | 22 +- src/core/ext/xds/xds_cluster.h | 2 +- .../ext/xds/xds_cluster_specifier_plugin.cc | 5 +- src/core/ext/xds/xds_health_status.cc | 14 +- src/core/ext/xds/xds_health_status.h | 6 +- src/core/ext/xds/xds_route_config.cc | 51 +- src/core/ext/xds/xds_route_config.h | 1 + src/core/ext/xds/xds_server_config_fetcher.cc | 50 +- .../credentials/xds/xds_credentials.cc | 49 +- .../credentials/xds/xds_credentials.h | 6 +- .../grpc_plugin_registry_extra.cc | 3 - src/python/grpcio/grpc_core_dependencies.py | 3 +- .../client_channel_service_config_test.cc | 31 +- .../lb_policy/lb_policy_test_lib.h | 41 +- ...xds_override_host_lb_config_parser_test.cc | 117 +- .../lb_policy/xds_override_host_test.cc | 86 +- ...tls_credentials_options_comparator_test.cc | 2 +- test/core/security/xds_credentials_test.cc | 19 +- .../core/xds/xds_certificate_provider_test.cc | 298 +--- .../xds/xds_cluster_resource_type_test.cc | 65 +- .../xds_route_config_resource_type_test.cc | 4 +- .../end2end/xds/xds_cluster_end2end_test.cc | 6 +- .../xds/xds_cluster_type_end2end_test.cc | 10 +- test/cpp/end2end/xds/xds_core_end2end_test.cc | 23 +- test/cpp/end2end/xds/xds_csds_end2end_test.cc | 2 +- .../xds/xds_fault_injection_end2end_test.cc | 3 + .../end2end/xds/xds_routing_end2end_test.cc | 4 + .../core/gen_grpc_tls_credentials_options.py | 2 +- tools/doxygen/Doxyfile.c++.internal | 7 +- tools/doxygen/Doxyfile.core.internal | 7 +- 60 files changed, 2945 insertions(+), 3125 deletions(-) delete mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc create mode 100644 src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc create mode 100644 src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h rename src/core/ext/filters/client_channel/resolver/xds/{xds_resolver.h => xds_resolver_attributes.h} (93%) create mode 100644 src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc create mode 100644 src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h diff --git a/BUILD b/BUILD index 8893aee2bb1..e244afae282 100644 --- a/BUILD +++ b/BUILD @@ -584,7 +584,6 @@ GRPC_XDS_TARGETS = [ "//src/core:grpc_lb_policy_cds", "//src/core:grpc_lb_policy_xds_cluster_impl", "//src/core:grpc_lb_policy_xds_cluster_manager", - "//src/core:grpc_lb_policy_xds_cluster_resolver", "//src/core:grpc_lb_policy_xds_override_host", "//src/core:grpc_lb_policy_xds_wrr_locality", "//src/core:grpc_lb_policy_ring_hash", diff --git a/CMakeLists.txt b/CMakeLists.txt index a35b971bd1b..d15ae57b0f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1793,7 +1793,6 @@ add_library(grpc src/core/ext/filters/client_channel/lb_policy/xds/cds.cc src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc src/core/ext/filters/client_channel/local_subchannel_pool.cc @@ -1812,7 +1811,9 @@ add_library(grpc src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc src/core/ext/filters/client_channel/resolver/polling_resolver.cc src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc src/core/ext/filters/client_channel/retry_filter.cc src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc src/core/ext/filters/client_channel/retry_service_config.cc diff --git a/Makefile b/Makefile index d9931302394..a1f5f107702 100644 --- a/Makefile +++ b/Makefile @@ -994,7 +994,6 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ @@ -1013,7 +1012,9 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc \ src/core/ext/filters/client_channel/resolver/polling_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ + src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc \ src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc \ + src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc \ src/core/ext/filters/client_channel/retry_filter.cc \ src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc \ src/core/ext/filters/client_channel/retry_service_config.cc \ @@ -3375,11 +3376,12 @@ src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc: $(OPENSSL_ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc: $(OPENSSL_DEP) -src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc: $(OPENSSL_DEP) +src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc: $(OPENSSL_DEP) src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc: $(OPENSSL_DEP) +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc: $(OPENSSL_DEP) src/core/ext/filters/rbac/rbac_filter.cc: $(OPENSSL_DEP) src/core/ext/filters/rbac/rbac_service_config_parser.cc: $(OPENSSL_DEP) src/core/ext/filters/server_config_selector/server_config_selector_filter.cc: $(OPENSSL_DEP) diff --git a/Package.swift b/Package.swift index 7fa98e64126..7212ca11453 100644 --- a/Package.swift +++ b/Package.swift @@ -189,7 +189,6 @@ let package = Package( "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h", "src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc", @@ -219,8 +218,12 @@ let package = Package( "src/core/ext/filters/client_channel/resolver/polling_resolver.cc", "src/core/ext/filters/client_channel/resolver/polling_resolver.h", "src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc", + "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc", + "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h", "src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc", - "src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h", + "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h", + "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc", + "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h", "src/core/ext/filters/client_channel/retry_filter.cc", "src/core/ext/filters/client_channel/retry_filter.h", "src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc", diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 2e7cd202f5c..462631306ab 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -263,7 +263,9 @@ libs: - src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.h - src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h - src/core/ext/filters/client_channel/resolver/polling_resolver.h - - src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h + - src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h + - src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h + - src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h - src/core/ext/filters/client_channel/retry_filter.h - src/core/ext/filters/client_channel/retry_filter_legacy_call_data.h - src/core/ext/filters/client_channel/retry_service_config.h @@ -1252,7 +1254,6 @@ libs: - src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc - - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc - src/core/ext/filters/client_channel/local_subchannel_pool.cc @@ -1271,7 +1272,9 @@ libs: - src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc - src/core/ext/filters/client_channel/resolver/polling_resolver.cc - src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + - src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc - src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + - src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc - src/core/ext/filters/client_channel/retry_filter.cc - src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc - src/core/ext/filters/client_channel/retry_service_config.cc diff --git a/config.m4 b/config.m4 index ad0d6caee28..c26543be07d 100644 --- a/config.m4 +++ b/config.m4 @@ -80,7 +80,6 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ @@ -99,7 +98,9 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc \ src/core/ext/filters/client_channel/resolver/polling_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ + src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc \ src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc \ + src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc \ src/core/ext/filters/client_channel/retry_filter.cc \ src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc \ src/core/ext/filters/client_channel/retry_service_config.cc \ diff --git a/config.w32 b/config.w32 index f18a8de275d..f2ed1de805b 100644 --- a/config.w32 +++ b/config.w32 @@ -45,7 +45,6 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\cds.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_cluster_impl.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_cluster_manager.cc " + - "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_cluster_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_override_host.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds_wrr_locality.cc " + "src\\core\\ext\\filters\\client_channel\\local_subchannel_pool.cc " + @@ -64,7 +63,9 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\resolver\\google_c2p\\google_c2p_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\polling_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\xds\\xds_dependency_manager.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\xds\\xds_resolver.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\xds\\xds_resolver_trace.cc " + "src\\core\\ext\\filters\\client_channel\\retry_filter.cc " + "src\\core\\ext\\filters\\client_channel\\retry_filter_legacy_call_data.cc " + "src\\core\\ext\\filters\\client_channel\\retry_service_config.cc " + diff --git a/doc/environment_variables.md b/doc/environment_variables.md index 41dbbd8426c..86db138e512 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -102,7 +102,6 @@ some configuration as environment variables that can be set. - xds_client - traces xds client - xds_cluster_manager_lb - traces cluster manager LB policy - xds_cluster_impl_lb - traces cluster impl LB policy - - xds_cluster_resolver_lb - traces xds cluster resolver LB policy - xds_resolver - traces xds resolver The following tracers will only run in binaries built in DEBUG mode. This is diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 83f07e2fe62..94b37911d90 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -294,7 +294,9 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.h', 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/resolver/polling_resolver.h', - 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h', 'src/core/ext/filters/client_channel/retry_filter.h', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.h', 'src/core/ext/filters/client_channel/retry_service_config.h', @@ -1552,7 +1554,9 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.h', 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/resolver/polling_resolver.h', - 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h', 'src/core/ext/filters/client_channel/retry_filter.h', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.h', 'src/core/ext/filters/client_channel/retry_service_config.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 64a415514b5..2784969c409 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -292,7 +292,6 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc', @@ -322,8 +321,12 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/polling_resolver.cc', 'src/core/ext/filters/client_channel/resolver/polling_resolver.h', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h', 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc', - 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h', 'src/core/ext/filters/client_channel/retry_filter.cc', 'src/core/ext/filters/client_channel/retry_filter.h', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc', @@ -2339,7 +2342,9 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.h', 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h', 'src/core/ext/filters/client_channel/resolver/polling_resolver.h', - 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h', 'src/core/ext/filters/client_channel/retry_filter.h', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.h', 'src/core/ext/filters/client_channel/retry_service_config.h', diff --git a/grpc.gemspec b/grpc.gemspec index 82993ca7ac9..ecbb77b1bb1 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,7 +195,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc ) - s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc ) @@ -225,8 +224,12 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/resolver/polling_resolver.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/polling_resolver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc ) - s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h ) + s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h ) + s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h ) s.files += %w( src/core/ext/filters/client_channel/retry_filter.cc ) s.files += %w( src/core/ext/filters/client_channel/retry_filter.h ) s.files += %w( src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc ) diff --git a/grpc.gyp b/grpc.gyp index 0cd7bdb032f..f8ba94de815 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -313,7 +313,6 @@ 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', @@ -332,7 +331,9 @@ 'src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc', 'src/core/ext/filters/client_channel/resolver/polling_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc', 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc', 'src/core/ext/filters/client_channel/retry_filter.cc', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc', 'src/core/ext/filters/client_channel/retry_service_config.cc', diff --git a/package.xml b/package.xml index 9ed1cf552fb..ea7259e4491 100644 --- a/package.xml +++ b/package.xml @@ -177,7 +177,6 @@ - @@ -207,8 +206,12 @@ + + - + + + diff --git a/src/core/BUILD b/src/core/BUILD index 63b603d6bc6..fccb572aa67 100644 --- a/src/core/BUILD +++ b/src/core/BUILD @@ -4187,7 +4187,7 @@ grpc_cc_library( "channel_args", "channel_fwd", "context", - "grpc_resolver_xds_header", + "grpc_resolver_xds_attributes", "grpc_service_config", "json", "json_args", @@ -4710,9 +4710,9 @@ grpc_cc_library( deps = [ "channel_args", "delegating_helper", - "grpc_matchers", + "grpc_lb_address_filtering", + "grpc_lb_xds_channel_args", "grpc_outlier_detection_header", - "grpc_tls_credentials", "grpc_xds_client", "json", "json_args", @@ -4725,6 +4725,7 @@ grpc_cc_library( "pollset_set", "time", "unique_type_name", + "xds_dependency_manager", "//:config", "//:debug_location", "//:gpr", @@ -4750,54 +4751,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc_lb_policy_xds_cluster_resolver", - srcs = [ - "ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc", - ], - external_deps = [ - "absl/functional:function_ref", - "absl/status", - "absl/status:statusor", - "absl/strings", - "absl/types:optional", - ], - language = "c++", - deps = [ - "channel_args", - "delegating_helper", - "grpc_lb_address_filtering", - "grpc_lb_xds_channel_args", - "grpc_xds_client", - "json", - "json_args", - "json_object_loader", - "json_writer", - "lb_policy", - "lb_policy_factory", - "lb_policy_registry", - "no_destruct", - "pollset_set", - "ref_counted_string", - "resolved_address", - "validation_errors", - "//:channel_arg_names", - "//:config", - "//:debug_location", - "//:endpoint_addresses", - "//:gpr", - "//:grpc_base", - "//:grpc_client_channel", - "//:grpc_resolver", - "//:grpc_resolver_fake", - "//:grpc_trace", - "//:orphanable", - "//:ref_counted_ptr", - "//:work_serializer", - "//:xds_client", - ], -) - grpc_cc_library( name = "grpc_lb_policy_xds_cluster_impl", srcs = [ @@ -4829,6 +4782,7 @@ grpc_cc_library( "resolved_address", "subchannel_interface", "validation_errors", + "xds_dependency_manager", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -4857,7 +4811,7 @@ grpc_cc_library( deps = [ "channel_args", "delegating_helper", - "grpc_resolver_xds_header", + "grpc_resolver_xds_attributes", "json", "json_args", "json_object_loader", @@ -5449,6 +5403,7 @@ grpc_cc_library( "resolved_address", "subchannel_interface", "validation_errors", + "xds_dependency_manager", "//:config", "//:debug_location", "//:endpoint_addresses", @@ -5758,9 +5713,9 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_resolver_xds_header", + name = "grpc_resolver_xds_attributes", hdrs = [ - "ext/filters/client_channel/resolver/xds/xds_resolver.h", + "ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h", ], external_deps = ["absl/strings"], language = "c++", @@ -5771,6 +5726,48 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_resolver_xds_trace", + srcs = [ + "ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc", + ], + hdrs = [ + "ext/filters/client_channel/resolver/xds/xds_resolver_trace.h", + ], + language = "c++", + deps = [ + "//:gpr_platform", + "//:grpc_trace", + ], +) + +grpc_cc_library( + name = "xds_dependency_manager", + srcs = [ + "ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc", + ], + hdrs = [ + "ext/filters/client_channel/resolver/xds/xds_dependency_manager.h", + ], + external_deps = [ + "absl/container:flat_hash_map", + "absl/container:flat_hash_set", + "absl/strings", + ], + language = "c++", + deps = [ + "grpc_lb_xds_channel_args", + "grpc_resolver_xds_trace", + "grpc_xds_client", + "match", + "ref_counted", + "//:config", + "//:gpr", + "//:grpc_resolver", + "//:grpc_resolver_fake", + ], +) + grpc_cc_library( name = "grpc_resolver_xds", srcs = [ @@ -5797,7 +5794,8 @@ grpc_cc_library( "dual_ref_counted", "experiments", "grpc_lb_policy_ring_hash", - "grpc_resolver_xds_header", + "grpc_resolver_xds_attributes", + "grpc_resolver_xds_trace", "grpc_service_config", "grpc_xds_client", "iomgr_fwd", @@ -5806,6 +5804,7 @@ grpc_cc_library( "ref_counted", "slice", "time", + "xds_dependency_manager", "xxhash_inline", "//:channel_arg_names", "//:config", diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc index 2aeae2e7a6b..4154adc5a70 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc @@ -37,11 +37,10 @@ #include #include +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" #include "src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h" -#include "src/core/ext/xds/certificate_provider_store.h" -#include "src/core/ext/xds/xds_certificate_provider.h" -#include "src/core/ext/xds/xds_client.h" -#include "src/core/ext/xds/xds_client_grpc.h" +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" #include "src/core/ext/xds/xds_cluster.h" #include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_health_status.h" @@ -64,11 +63,6 @@ #include "src/core/lib/load_balancing/lb_policy.h" #include "src/core/lib/load_balancing/lb_policy_factory.h" #include "src/core/lib/load_balancing/lb_policy_registry.h" -#include "src/core/lib/matchers/matchers.h" -#include "src/core/lib/security/credentials/credentials.h" -#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" -#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" -#include "src/core/lib/security/credentials/xds/xds_credentials.h" namespace grpc_core { @@ -76,9 +70,9 @@ TraceFlag grpc_cds_lb_trace(false, "cds_lb"); namespace { -constexpr absl::string_view kCds = "cds_experimental"; +using XdsConfig = XdsDependencyManager::XdsConfig; -constexpr int kMaxAggregateClusterRecursionDepth = 16; +constexpr absl::string_view kCds = "cds_experimental"; // Config for this LB policy. class CdsLbConfig : public LoadBalancingPolicy::Config { @@ -91,24 +85,29 @@ class CdsLbConfig : public LoadBalancingPolicy::Config { CdsLbConfig(CdsLbConfig&& other) = delete; CdsLbConfig& operator=(CdsLbConfig&& other) = delete; - const std::string& cluster() const { return cluster_; } absl::string_view name() const override { return kCds; } + const std::string& cluster() const { return cluster_; } + bool is_dynamic() const { return is_dynamic_; } + static const JsonLoaderInterface* JsonLoader(const JsonArgs&) { - static const auto* loader = JsonObjectLoader() - .Field("cluster", &CdsLbConfig::cluster_) - .Finish(); + static const auto* loader = + JsonObjectLoader() + .Field("cluster", &CdsLbConfig::cluster_) + .OptionalField("isDynamic", &CdsLbConfig::is_dynamic_) + .Finish(); return loader; } private: std::string cluster_; + bool is_dynamic_ = false; }; // CDS LB policy. class CdsLb : public LoadBalancingPolicy { public: - CdsLb(RefCountedPtr xds_client, Args args); + explicit CdsLb(Args args); absl::string_view name() const override { return kCds; } @@ -117,109 +116,50 @@ class CdsLb : public LoadBalancingPolicy { void ExitIdleLocked() override; private: - // Watcher for getting cluster data from XdsClient. - class ClusterWatcher : public XdsClusterResourceType::WatcherInterface { - public: - ClusterWatcher(RefCountedPtr parent, std::string name) - : parent_(std::move(parent)), name_(std::move(name)) {} - - void OnResourceChanged( - std::shared_ptr cluster_data, - RefCountedPtr read_delay_handle) override { - parent_->work_serializer()->Run( - [self = RefAsSubclass(), - cluster_data = std::move(cluster_data), - read_handle = std::move(read_delay_handle)]() mutable { - self->parent_->OnClusterChanged(self->name_, - std::move(cluster_data)); - }, - DEBUG_LOCATION); - } - void OnError( - absl::Status status, - RefCountedPtr read_delay_handle) override { - parent_->work_serializer()->Run( - [self = RefAsSubclass(), status = std::move(status), - read_handle = std::move(read_delay_handle)]() mutable { - self->parent_->OnError(self->name_, std::move(status)); - }, - DEBUG_LOCATION); - } - void OnResourceDoesNotExist( - RefCountedPtr read_delay_handle) override { - parent_->work_serializer()->Run( - [self = RefAsSubclass(), - read_handle = std::move(read_delay_handle)]() { - self->parent_->OnResourceDoesNotExist(self->name_); - }, - DEBUG_LOCATION); - } - - private: - RefCountedPtr parent_; - std::string name_; - }; - - struct WatcherState { - // Pointer to watcher, to be used when cancelling. - // Not owned, so do not dereference. - ClusterWatcher* watcher = nullptr; - // Most recent update obtained from this watcher. - std::shared_ptr update; - }; - // Delegating helper to be passed to child policy. using Helper = ParentOwningDelegatingChannelControlHelper; + // State used to retain child policy names for the priority policy. + struct ChildNameState { + std::vector priority_child_numbers; + size_t next_available_child_number = 0; + }; + ~CdsLb() override; void ShutdownLocked() override; - absl::StatusOr GenerateDiscoveryMechanismForCluster( - const std::string& name, int depth, Json::Array* discovery_mechanisms, - std::set* clusters_added); - void OnClusterChanged(const std::string& name, - std::shared_ptr cluster_data); - void OnError(const std::string& name, absl::Status status); - void OnResourceDoesNotExist(const std::string& name); + // Computes child numbers for new_cluster_list, reusing child numbers + // from old_cluster_list and child_name_state_list_ in an intelligent + // way to avoid unnecessary churn. + std::vector ComputeChildNames( + const std::vector& old_cluster_list, + const std::vector& new_cluster_list) + const; - absl::Status UpdateXdsCertificateProvider( - const std::string& cluster_name, const XdsClusterResource& cluster_data); + std::string GetChildPolicyName(const std::string& cluster, size_t priority); - void CancelClusterDataWatch(absl::string_view cluster_name, - ClusterWatcher* watcher, - bool delay_unsubscription = false); + Json CreateChildPolicyConfig( + const Json::Array& lb_policy_config, + const std::vector& new_cluster_list); + std::shared_ptr CreateChildPolicyAddresses( + const std::vector& new_cluster_list); + std::string CreateChildPolicyResolutionNote( + const std::vector& new_cluster_list); - void MaybeDestroyChildPolicyLocked(); + void ResetState(); - RefCountedPtr config_; + void ReportTransientFailure(absl::Status status); - // Current channel args from the resolver. - ChannelArgs args_; + std::string cluster_name_; + RefCountedPtr xds_config_; - // The xds client. - RefCountedPtr xds_client_; + // Cluster subscription, for dynamic clusters (e.g., RLS). + RefCountedPtr subscription_; - // Maps from cluster name to the state for that cluster. - // The root of the tree is config_->cluster(). - std::map watchers_; - - // TODO(roth, yashkt): These are here because XdsCertificateProvider - // does not store the actual underlying cert providers, it stores only - // their distributors, so we need to hold a ref to the cert providers - // here. However, in the aggregate cluster case, there may be multiple - // clusters in the same cert provider, and we're only tracking the cert - // providers for the most recent underlying cluster here. This is - // clearly a bug, and I think it will cause us to stop getting updates - // for all but one of the cert providers in the aggregate cluster - // case. Need to figure out the right way to fix this -- I don't - // think we want to store another map here, so ideally, we should just - // have XdsCertificateProvider actually hold the refs to the cert - // providers instead of just the distributors. - RefCountedPtr root_certificate_provider_; - RefCountedPtr identity_certificate_provider_; - - RefCountedPtr xds_certificate_provider_; + // The elements in this vector correspond to those in + // xds_config_->clusters[cluster_name_]. + std::vector child_name_state_list_; // Child LB policy. OrphanablePtr child_policy_; @@ -232,11 +172,9 @@ class CdsLb : public LoadBalancingPolicy { // CdsLb // -CdsLb::CdsLb(RefCountedPtr xds_client, Args args) - : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { +CdsLb::CdsLb(Args args) : LoadBalancingPolicy(std::move(args)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] created -- using xds client %p", this, - xds_client_.get()); + gpr_log(GPR_INFO, "[cdslb %p] created", this); } } @@ -251,28 +189,7 @@ void CdsLb::ShutdownLocked() { gpr_log(GPR_INFO, "[cdslb %p] shutting down", this); } shutting_down_ = true; - MaybeDestroyChildPolicyLocked(); - if (xds_client_ != nullptr) { - for (auto& watcher : watchers_) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] cancelling watch for cluster %s", this, - watcher.first.c_str()); - } - CancelClusterDataWatch(watcher.first, watcher.second.watcher, - /*delay_unsubscription=*/false); - } - watchers_.clear(); - xds_client_.reset(DEBUG_LOCATION, "CdsLb"); - } - args_ = ChannelArgs(); -} - -void CdsLb::MaybeDestroyChildPolicyLocked() { - if (child_policy_ != nullptr) { - grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), - interested_parties()); - child_policy_.reset(); - } + ResetState(); } void CdsLb::ResetBackoffLocked() { @@ -283,390 +200,552 @@ void CdsLb::ExitIdleLocked() { if (child_policy_ != nullptr) child_policy_->ExitIdleLocked(); } -absl::Status CdsLb::UpdateLocked(UpdateArgs args) { - // Update config. - auto old_config = std::move(config_); - config_ = args.config.TakeAsSubclass(); - if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] received update: cluster=%s", this, - config_->cluster().c_str()); - } - // Update args. - args_ = std::move(args.args); - // If cluster name changed, cancel watcher and restart. - if (old_config == nullptr || old_config->cluster() != config_->cluster()) { - if (old_config != nullptr) { - for (auto& watcher : watchers_) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] cancelling watch for cluster %s", this, - watcher.first.c_str()); - } - CancelClusterDataWatch(watcher.first, watcher.second.watcher, - /*delay_unsubscription=*/true); - } - watchers_.clear(); - } - auto watcher = MakeRefCounted(RefAsSubclass(), - config_->cluster()); - watchers_[config_->cluster()].watcher = watcher.get(); - XdsClusterResourceType::StartWatch(xds_client_.get(), config_->cluster(), - std::move(watcher)); +// We need at least one priority for each discovery mechanism, just so that we +// have a child in which to create the xds_cluster_impl policy. This ensures +// that we properly handle the case of a discovery mechanism dropping 100% of +// calls, the OnError() case, and the OnResourceDoesNotExist() case. +const XdsEndpointResource::PriorityList& GetUpdatePriorityList( + const XdsEndpointResource* update) { + static const NoDestruct + kPriorityListWithEmptyPriority(1); + if (update == nullptr || update->priorities.empty()) { + return *kPriorityListWithEmptyPriority; } - return absl::OkStatus(); + return update->priorities; } -// Generates the discovery mechanism config for the specified cluster name. -// -// If no CDS update has been received for the cluster, starts the watcher -// if needed, and returns false. Otherwise, generates the discovery -// mechanism config, adds it to *discovery_mechanisms, and returns true. -// -// For aggregate clusters, may call itself recursively. Returns an -// error if depth exceeds kMaxAggregateClusterRecursionDepth. -absl::StatusOr CdsLb::GenerateDiscoveryMechanismForCluster( - const std::string& name, int depth, Json::Array* discovery_mechanisms, - std::set* clusters_added) { - if (depth == kMaxAggregateClusterRecursionDepth) { - return absl::FailedPreconditionError( - "aggregate cluster graph exceeds max depth"); - } - if (!clusters_added->insert(name).second) { - return true; // Discovery mechanism already added from some other branch. +absl::StatusOr> +BuildLeafClusterConfigList(const XdsConfig* xds_config, + const XdsConfig::ClusterConfig* cluster_config) { + if (cluster_config == nullptr) { + return std::vector(); } - auto& state = watchers_[name]; - // Create a new watcher if needed. - if (state.watcher == nullptr) { - auto watcher = MakeRefCounted(RefAsSubclass(), name); - if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] starting watch for cluster %s", this, - name.c_str()); - } - state.watcher = watcher.get(); - XdsClusterResourceType::StartWatch(xds_client_.get(), name, - std::move(watcher)); - return false; - } - // Don't have the update we need yet. - if (state.update == nullptr) return false; - // For AGGREGATE clusters, recursively expand to child clusters. - auto* aggregate = - absl::get_if(&state.update->type); - if (aggregate != nullptr) { - bool missing_cluster = false; - for (const std::string& child_name : aggregate->prioritized_cluster_names) { - auto result = GenerateDiscoveryMechanismForCluster( - child_name, depth + 1, discovery_mechanisms, clusters_added); - if (!result.ok()) return result; - if (!*result) missing_cluster = true; - } - return !missing_cluster; - } - Json::Object mechanism = { - {"clusterName", Json::FromString(name)}, - {"max_concurrent_requests", - Json::FromNumber(state.update->max_concurrent_requests)}, - }; - if (state.update->outlier_detection.has_value()) { - auto& outlier_detection_update = state.update->outlier_detection.value(); - Json::Object outlier_detection; - outlier_detection["interval"] = - Json::FromString(outlier_detection_update.interval.ToJsonString()); - outlier_detection["baseEjectionTime"] = Json::FromString( - outlier_detection_update.base_ejection_time.ToJsonString()); - outlier_detection["maxEjectionTime"] = Json::FromString( - outlier_detection_update.max_ejection_time.ToJsonString()); - outlier_detection["maxEjectionPercent"] = - Json::FromNumber(outlier_detection_update.max_ejection_percent); - if (outlier_detection_update.success_rate_ejection.has_value()) { - outlier_detection["successRateEjection"] = Json::FromObject({ - {"stdevFactor", - Json::FromNumber( - outlier_detection_update.success_rate_ejection->stdev_factor)}, - {"enforcementPercentage", - Json::FromNumber(outlier_detection_update.success_rate_ejection - ->enforcement_percentage)}, - {"minimumHosts", - Json::FromNumber( - outlier_detection_update.success_rate_ejection->minimum_hosts)}, - {"requestVolume", - Json::FromNumber( - outlier_detection_update.success_rate_ejection->request_volume)}, - }); - } - if (outlier_detection_update.failure_percentage_ejection.has_value()) { - outlier_detection["failurePercentageEjection"] = Json::FromObject({ - {"threshold", - Json::FromNumber(outlier_detection_update - .failure_percentage_ejection->threshold)}, - {"enforcementPercentage", - Json::FromNumber( - outlier_detection_update.failure_percentage_ejection - ->enforcement_percentage)}, - {"minimumHosts", - Json::FromNumber(outlier_detection_update - .failure_percentage_ejection->minimum_hosts)}, - {"requestVolume", - Json::FromNumber(outlier_detection_update - .failure_percentage_ejection->request_volume)}, + GPR_ASSERT(xds_config != nullptr); + std::vector tmp_leaf_clusters; + const std::vector& leaf_clusters = Match( + cluster_config->children, + [&](const XdsConfig::ClusterConfig::EndpointConfig&) { + tmp_leaf_clusters.push_back(cluster_config->cluster_name); + return tmp_leaf_clusters; + }, + [&](const XdsConfig::ClusterConfig::AggregateConfig& aggregate_config) { + return aggregate_config.leaf_clusters; }); + std::vector leaf_cluster_configs; + leaf_cluster_configs.reserve(leaf_clusters.size()); + for (const absl::string_view cluster_name : leaf_clusters) { + auto it = xds_config->clusters.find(cluster_name); + if (it == xds_config->clusters.end() || !it->second.ok() || + it->second->cluster == nullptr) { + return absl::InternalError(absl::StrCat( + "xDS config does not contain an entry for cluster ", cluster_name)); } - mechanism["outlierDetection"] = - Json::FromObject(std::move(outlier_detection)); - } - Match( - state.update->type, - [&](const XdsClusterResource::Eds& eds) { - mechanism["type"] = Json::FromString("EDS"); - if (!eds.eds_service_name.empty()) { - mechanism["edsServiceName"] = Json::FromString(eds.eds_service_name); - } - }, - [&](const XdsClusterResource::LogicalDns& logical_dns) { - mechanism["type"] = Json::FromString("LOGICAL_DNS"); - mechanism["dnsHostname"] = Json::FromString(logical_dns.hostname); - }, - [&](const XdsClusterResource::Aggregate&) { GPR_ASSERT(0); }); - if (state.update->lrs_load_reporting_server.has_value()) { - mechanism["lrsLoadReportingServer"] = - state.update->lrs_load_reporting_server->ToJson(); - } - if (!state.update->override_host_statuses.empty()) { - Json::Array status_list; - for (const auto& status : state.update->override_host_statuses) { - status_list.emplace_back(Json::FromString(status.ToString())); + const XdsConfig::ClusterConfig& cluster_config = *it->second; + if (!absl::holds_alternative( + cluster_config.children)) { + return absl::InternalError(absl::StrCat("xDS config entry for cluster ", + cluster_name, + " has no endpoint config")); } - mechanism["overrideHostStatus"] = Json::FromArray(std::move(status_list)); + leaf_cluster_configs.push_back(&cluster_config); } - discovery_mechanisms->emplace_back(Json::FromObject(std::move(mechanism))); - return true; + return leaf_cluster_configs; } -void CdsLb::OnClusterChanged( - const std::string& name, - std::shared_ptr cluster_data) { +absl::Status CdsLb::UpdateLocked(UpdateArgs args) { + // Get new config. + auto new_config = args.config.TakeAsSubclass(); if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log( - GPR_INFO, - "[cdslb %p] received CDS update for cluster %s from xds client %p: %s", - this, name.c_str(), xds_client_.get(), - cluster_data->ToString().c_str()); + gpr_log(GPR_INFO, "[cdslb %p] received update: cluster=%s is_dynamic=%d", + this, new_config->cluster().c_str(), new_config->is_dynamic()); } - // Store the update in the map if we are still interested in watching this - // cluster (i.e., it is not cancelled already). - // If we've already deleted this entry, then this is an update notification - // that was scheduled before the deletion, so we can just ignore it. - auto it = watchers_.find(name); - if (it == watchers_.end()) return; - it->second.update = std::move(cluster_data); - // Take care of integration with new certificate code. - absl::Status status = UpdateXdsCertificateProvider(name, *it->second.update); - if (!status.ok()) { - return OnError(name, status); + GPR_ASSERT(new_config != nullptr); + // Cluster name should never change, because we should use a different + // child name in xds_cluster_manager in that case. + if (cluster_name_.empty()) { + cluster_name_ = new_config->cluster(); + } else { + GPR_ASSERT(cluster_name_ == new_config->cluster()); } - // Scan the map starting from the root cluster to generate the list of - // discovery mechanisms. If we don't have some of the data we need (i.e., we - // just started up and not all watchers have returned data yet), then don't - // update the child policy at all. - Json::Array discovery_mechanisms; - std::set clusters_added; - auto result = GenerateDiscoveryMechanismForCluster( - config_->cluster(), /*depth=*/0, &discovery_mechanisms, &clusters_added); - if (!result.ok()) { - return OnError(name, result.status()); + // Get xDS config. + auto new_xds_config = args.args.GetObjectRef(); + if (new_xds_config == nullptr) { + // Should never happen. + absl::Status status = + absl::InternalError("xDS config not passed to CDS LB policy"); + ReportTransientFailure(status); + return status; } - if (*result) { - if (discovery_mechanisms.empty()) { - return OnError(name, absl::FailedPreconditionError( - "aggregate cluster graph has no leaf clusters")); - } - // LB policy is configured by aggregate cluster, not by the individual - // underlying cluster that we may be processing an update for. - auto it = watchers_.find(config_->cluster()); - GPR_ASSERT(it != watchers_.end()); - // Construct config for child policy. - Json json = Json::FromArray({ - Json::FromObject({ - {"xds_cluster_resolver_experimental", - Json::FromObject({ - {"xdsLbPolicy", - Json::FromArray(it->second.update->lb_policy_config)}, - {"discoveryMechanisms", - Json::FromArray(std::move(discovery_mechanisms))}, - })}, - }), - }); - if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] generated config for child policy: %s", - this, JsonDump(json, /*indent=*/1).c_str()); - } - auto config = - CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( - json); - if (!config.ok()) { - OnError(name, absl::UnavailableError(config.status().message())); - return; - } - // Create child policy if not already present. - if (child_policy_ == nullptr) { - LoadBalancingPolicy::Args args; - args.work_serializer = work_serializer(); - args.args = args_; - args.channel_control_helper = - std::make_unique(RefAsSubclass()); - child_policy_ = - CoreConfiguration::Get() - .lb_policy_registry() - .CreateLoadBalancingPolicy((*config)->name(), std::move(args)); - if (child_policy_ == nullptr) { - OnError(name, absl::UnavailableError("failed to create child policy")); - return; + auto it = new_xds_config->clusters.find(cluster_name_); + if (it == new_xds_config->clusters.end()) { + // Cluster not present. + if (new_config->is_dynamic()) { + // This is a dynamic cluster. Subscribe to it if not yet subscribed. + if (subscription_ == nullptr) { + auto* dependency_mgr = args.args.GetObject(); + if (dependency_mgr == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError( + "xDS dependency mgr not passed to CDS LB policy"); + ReportTransientFailure(status); + return status; + } + subscription_ = dependency_mgr->GetClusterSubscription(cluster_name_); + // Stay in CONNECTING until we get an update that has the cluster. + return absl::OkStatus(); } - grpc_pollset_set_add_pollset_set(child_policy_->interested_parties(), - interested_parties()); + // If we are already subscribed, it's possible that we just + // recently subscribed but another update came through before we + // got the new cluster, in which case it will still be missing. if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] created child policy %s (%p)", this, - std::string((*config)->name()).c_str(), child_policy_.get()); + gpr_log(GPR_INFO, + "[cdslb %p] xDS config has no entry for dynamic cluster %s, " + "ignoring update", + this, cluster_name_.c_str()); } + // Stay in CONNECTING until we get an update that has the cluster. + return absl::OkStatus(); } - // Update child policy. - UpdateArgs args; - args.config = std::move(*config); - if (xds_certificate_provider_ != nullptr) { - args.args = args_.SetObject(xds_certificate_provider_); - } else { - args.args = args_; + // Not a dynamic cluster. This should never happen. + absl::Status status = absl::UnavailableError(absl::StrCat( + "xDS config has no entry for static cluster ", cluster_name_)); + ReportTransientFailure(status); + return status; + } + auto& new_cluster_config = it->second; + // If new config is not OK, report TRANSIENT_FAILURE. + if (!new_cluster_config.ok()) { + ReportTransientFailure(new_cluster_config.status()); + return new_cluster_config.status(); + } + GPR_ASSERT(new_cluster_config->cluster != nullptr); + // Find old cluster config, if any. + const XdsConfig::ClusterConfig* old_cluster_config = nullptr; + if (xds_config_ != nullptr) { + auto it_old = xds_config_->clusters.find(cluster_name_); + if (it_old != xds_config_->clusters.end() && it_old->second.ok()) { + old_cluster_config = &*it_old->second; + // If nothing changed for a leaf cluster, then ignore the update. + // Can't do this for an aggregate cluster, because even if the aggregate + // cluster itself didn't change, the leaf clusters may have changed. + if (*new_cluster_config == *old_cluster_config && + absl::holds_alternative( + new_cluster_config->children)) { + return absl::OkStatus(); + } } - // TODO(roth): If the child policy reports an error with the update, - // we need to propagate the error to the resolver somehow. - (void)child_policy_->UpdateLocked(std::move(args)); } - // Remove entries in watchers_ for any clusters not in clusters_added - for (auto it = watchers_.begin(); it != watchers_.end();) { - const std::string& cluster_name = it->first; - if (clusters_added.find(cluster_name) != clusters_added.end()) { - ++it; - continue; + // Construct lists of old and new leaf cluster configs. + auto old_leaf_cluster_configs = + BuildLeafClusterConfigList(xds_config_.get(), old_cluster_config); + if (!old_leaf_cluster_configs.ok()) { + ReportTransientFailure(old_leaf_cluster_configs.status()); + return old_leaf_cluster_configs.status(); + } + auto new_leaf_cluster_configs = + BuildLeafClusterConfigList(new_xds_config.get(), &*new_cluster_config); + if (!new_leaf_cluster_configs.ok()) { + ReportTransientFailure(new_leaf_cluster_configs.status()); + return new_leaf_cluster_configs.status(); + } + // Swap in new config and compute new child numbers. + child_name_state_list_ = + ComputeChildNames(*old_leaf_cluster_configs, *new_leaf_cluster_configs); + xds_config_ = std::move(new_xds_config); + // Construct child policy config. + Json json = CreateChildPolicyConfig( + new_cluster_config->cluster->lb_policy_config, *new_leaf_cluster_configs); + auto child_config = + CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( + json); + if (!child_config.ok()) { + // Should never happen. + absl::Status status = absl::InternalError( + absl::StrCat(cluster_name_, ": error parsing child policy config: ", + child_config.status().message())); + ReportTransientFailure(status); + return status; + } + // Create child policy if not already present. + if (child_policy_ == nullptr) { + LoadBalancingPolicy::Args lb_args; + lb_args.work_serializer = work_serializer(); + lb_args.args = args.args; + lb_args.channel_control_helper = + std::make_unique(RefAsSubclass()); + child_policy_ = + CoreConfiguration::Get().lb_policy_registry().CreateLoadBalancingPolicy( + (*child_config)->name(), std::move(lb_args)); + if (child_policy_ == nullptr) { + // Should never happen. + absl::Status status = absl::UnavailableError( + absl::StrCat(cluster_name_, ": failed to create child policy")); + ReportTransientFailure(status); + return status; } + grpc_pollset_set_add_pollset_set(child_policy_->interested_parties(), + interested_parties()); if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] cancelling watch for cluster %s", this, - cluster_name.c_str()); + gpr_log(GPR_INFO, "[cdslb %p] created child policy %s (%p)", this, + std::string((*child_config)->name()).c_str(), + child_policy_.get()); } - CancelClusterDataWatch(cluster_name, it->second.watcher, - /*delay_unsubscription=*/false); - it = watchers_.erase(it); } + // Update child policy. + UpdateArgs update_args; + update_args.config = std::move(*child_config); + update_args.addresses = CreateChildPolicyAddresses(*new_leaf_cluster_configs); + update_args.resolution_note = + CreateChildPolicyResolutionNote(*new_leaf_cluster_configs); + update_args.args = args.args; + return child_policy_->UpdateLocked(std::move(update_args)); } -void CdsLb::OnError(const std::string& name, absl::Status status) { - gpr_log(GPR_ERROR, "[cdslb %p] xds error obtaining data for cluster %s: %s", - this, name.c_str(), status.ToString().c_str()); - // Go into TRANSIENT_FAILURE if we have not yet created the child - // policy (i.e., we have not yet received data from xds). Otherwise, - // we keep running with the data we had previously. - if (child_policy_ == nullptr) { - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(absl::UnavailableError( - absl::StrCat(name, ": ", status.ToString())))); +std::vector CdsLb::ComputeChildNames( + const std::vector& old_cluster_list, + const std::vector& new_cluster_list) + const { + // First, build some maps from locality to child number and the reverse + // from old_cluster_list and child_name_state_list_. + struct LocalityChildNumberMapping { + std::map + locality_child_map; + std::map> + child_locality_map; + size_t next_available_child_number; + }; + std::map cluster_mappings; + size_t old_index = 0; + for (const auto* cluster_config : old_cluster_list) { + GPR_ASSERT(old_index < child_name_state_list_.size()); + const auto& old_numbers = child_name_state_list_[old_index++]; + const auto& endpoint_config = + absl::get( + cluster_config->children); + const auto& prev_priority_list = + GetUpdatePriorityList(endpoint_config.endpoints.get()); + auto& mappings = cluster_mappings[cluster_config->cluster_name]; + mappings.next_available_child_number = + old_numbers.next_available_child_number; + for (size_t priority = 0; priority < prev_priority_list.size(); + ++priority) { + size_t child_number = old_numbers.priority_child_numbers[priority]; + const auto& localities = prev_priority_list[priority].localities; + for (const auto& p : localities) { + XdsLocalityName* locality_name = p.first; + mappings.locality_child_map[locality_name] = child_number; + mappings.child_locality_map[child_number].insert(locality_name); + } + } } + // Now construct a new list containing priority child numbers for the new + // list based on cluster_mappings. + std::vector new_numbers_list; + for (const auto* cluster_config : new_cluster_list) { + auto& mappings = cluster_mappings[cluster_config->cluster_name]; + new_numbers_list.emplace_back(); + auto& new_numbers = new_numbers_list.back(); + new_numbers.next_available_child_number = + mappings.next_available_child_number; + const auto& endpoint_config = + absl::get( + cluster_config->children); + const XdsEndpointResource::PriorityList& priority_list = + GetUpdatePriorityList(endpoint_config.endpoints.get()); + for (size_t priority = 0; priority < priority_list.size(); ++priority) { + const auto& localities = priority_list[priority].localities; + absl::optional child_number; + // If one of the localities in this priority already existed, reuse its + // child number. + for (const auto& p : localities) { + XdsLocalityName* locality_name = p.first; + if (!child_number.has_value()) { + auto it = mappings.locality_child_map.find(locality_name); + if (it != mappings.locality_child_map.end()) { + child_number = it->second; + mappings.locality_child_map.erase(it); + // Remove localities that *used* to be in this child number, so + // that we don't incorrectly reuse this child number for a + // subsequent priority. + for (XdsLocalityName* old_locality : + mappings.child_locality_map[*child_number]) { + mappings.locality_child_map.erase(old_locality); + } + } + } else { + // Remove all localities that are now in this child number, so + // that we don't accidentally reuse this child number for a + // subsequent priority. + mappings.locality_child_map.erase(locality_name); + } + } + // If we didn't find an existing child number, assign a new one. + if (!child_number.has_value()) { + for (child_number = new_numbers.next_available_child_number; + mappings.child_locality_map.find(*child_number) != + mappings.child_locality_map.end(); + ++(*child_number)) { + } + new_numbers.next_available_child_number = *child_number + 1; + // Add entry so we know that the child number is in use. + // (Don't need to add the list of localities, since we won't use them.) + mappings.child_locality_map[*child_number]; + } + new_numbers.priority_child_numbers.push_back(*child_number); + } + } + return new_numbers_list; } -void CdsLb::OnResourceDoesNotExist(const std::string& name) { - gpr_log(GPR_ERROR, - "[cdslb %p] CDS resource for %s does not exist -- reporting " - "TRANSIENT_FAILURE", - this, name.c_str()); - absl::Status status = absl::UnavailableError( - absl::StrCat("CDS resource \"", config_->cluster(), "\" does not exist")); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(status)); - MaybeDestroyChildPolicyLocked(); +std::string MakeChildPolicyName(absl::string_view cluster, + size_t child_number) { + return absl::StrCat("{cluster=", cluster, ", child_number=", child_number, + "}"); } -absl::Status CdsLb::UpdateXdsCertificateProvider( - const std::string& cluster_name, const XdsClusterResource& cluster_data) { - // Early out if channel is not configured to use xds security. - auto channel_credentials = channel_control_helper()->GetChannelCredentials(); - if (channel_credentials == nullptr || - channel_credentials->type() != XdsCredentials::Type()) { - xds_certificate_provider_ = nullptr; - return absl::OkStatus(); +Json CdsLb::CreateChildPolicyConfig( + const Json::Array& lb_policy_config, + const std::vector& new_cluster_list) { + Json::Object priority_children; + Json::Array priority_priorities; + size_t numbers_index = 0; + for (const auto* cluster_config : new_cluster_list) { + const bool is_logical_dns = + absl::holds_alternative( + cluster_config->cluster->type); + const auto& endpoint_config = + absl::get( + cluster_config->children); + const auto& priority_list = + GetUpdatePriorityList(endpoint_config.endpoints.get()); + const auto& cluster_resource = *cluster_config->cluster; + GPR_ASSERT(numbers_index < child_name_state_list_.size()); + const auto& child_numbers = child_name_state_list_[numbers_index++]; + for (size_t priority = 0; priority < priority_list.size(); ++priority) { + // Determine what xDS LB policy to use. + Json child_policy; + if (is_logical_dns) { + child_policy = Json::FromArray({ + Json::FromObject({ + {"pick_first", Json::FromObject({})}, + }), + }); + } else { + child_policy = Json::FromArray(lb_policy_config); + } + // Wrap the xDS LB policy in the xds_override_host policy. + Json::Object xds_override_host_lb_config = { + {"clusterName", Json::FromString(cluster_config->cluster_name)}, + {"childPolicy", std::move(child_policy)}, + }; + Json::Array xds_override_host_config = {Json::FromObject({ + {"xds_override_host_experimental", + Json::FromObject(std::move(xds_override_host_lb_config))}, + })}; + // Wrap it in the xds_cluster_impl policy. + Json::Array xds_cluster_impl_config = {Json::FromObject( + {{"xds_cluster_impl_experimental", + Json::FromObject({ + {"clusterName", Json::FromString(cluster_config->cluster_name)}, + {"childPolicy", + Json::FromArray(std::move(xds_override_host_config))}, + })}})}; + // Wrap it in the outlier_detection policy. + Json::Object outlier_detection_config; + if (cluster_resource.outlier_detection.has_value()) { + auto& outlier_detection_update = *cluster_resource.outlier_detection; + outlier_detection_config["interval"] = + Json::FromString(outlier_detection_update.interval.ToJsonString()); + outlier_detection_config["baseEjectionTime"] = Json::FromString( + outlier_detection_update.base_ejection_time.ToJsonString()); + outlier_detection_config["maxEjectionTime"] = Json::FromString( + outlier_detection_update.max_ejection_time.ToJsonString()); + outlier_detection_config["maxEjectionPercent"] = + Json::FromNumber(outlier_detection_update.max_ejection_percent); + if (outlier_detection_update.success_rate_ejection.has_value()) { + outlier_detection_config["successRateEjection"] = Json::FromObject({ + {"stdevFactor", + Json::FromNumber(outlier_detection_update.success_rate_ejection + ->stdev_factor)}, + {"enforcementPercentage", + Json::FromNumber(outlier_detection_update.success_rate_ejection + ->enforcement_percentage)}, + {"minimumHosts", + Json::FromNumber(outlier_detection_update.success_rate_ejection + ->minimum_hosts)}, + {"requestVolume", + Json::FromNumber(outlier_detection_update.success_rate_ejection + ->request_volume)}, + }); + } + if (outlier_detection_update.failure_percentage_ejection.has_value()) { + outlier_detection_config["failurePercentageEjection"] = + Json::FromObject({ + {"threshold", + Json::FromNumber( + outlier_detection_update.failure_percentage_ejection + ->threshold)}, + {"enforcementPercentage", + Json::FromNumber( + outlier_detection_update.failure_percentage_ejection + ->enforcement_percentage)}, + {"minimumHosts", + Json::FromNumber( + outlier_detection_update.failure_percentage_ejection + ->minimum_hosts)}, + {"requestVolume", + Json::FromNumber( + outlier_detection_update.failure_percentage_ejection + ->request_volume)}, + }); + } + } + outlier_detection_config["childPolicy"] = + Json::FromArray(std::move(xds_cluster_impl_config)); + Json locality_picking_policy = Json::FromArray({Json::FromObject({ + {"outlier_detection_experimental", + Json::FromObject(std::move(outlier_detection_config))}, + })}); + // Add priority entry, with the appropriate child name. + std::string child_name = + MakeChildPolicyName(cluster_config->cluster_name, + child_numbers.priority_child_numbers[priority]); + priority_priorities.emplace_back(Json::FromString(child_name)); + Json::Object child_config = { + {"config", std::move(locality_picking_policy)}, + }; + if (!is_logical_dns) { + child_config["ignore_reresolution_requests"] = Json::FromBool(true); + } + priority_children[child_name] = Json::FromObject(std::move(child_config)); + } } - if (xds_certificate_provider_ == nullptr) { - xds_certificate_provider_ = MakeRefCounted(); + Json json = Json::FromArray({Json::FromObject({ + {"priority_experimental", + Json::FromObject({ + {"children", Json::FromObject(std::move(priority_children))}, + {"priorities", Json::FromArray(std::move(priority_priorities))}, + })}, + })}); + if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { + gpr_log(GPR_INFO, "[cdslb %p] generated config for child policy: %s", this, + JsonDump(json, /*indent=*/1).c_str()); } - // Configure root cert. - absl::string_view root_provider_instance_name = - cluster_data.common_tls_context.certificate_validation_context - .ca_certificate_provider_instance.instance_name; - absl::string_view root_provider_cert_name = - cluster_data.common_tls_context.certificate_validation_context - .ca_certificate_provider_instance.certificate_name; - RefCountedPtr new_root_provider; - if (!root_provider_instance_name.empty()) { - new_root_provider = - xds_client_->certificate_provider_store() - .CreateOrGetCertificateProvider(root_provider_instance_name); - if (new_root_provider == nullptr) { - return absl::UnavailableError( - absl::StrCat("Certificate provider instance name: \"", - root_provider_instance_name, "\" not recognized.")); + return json; +} + +class PriorityEndpointIterator : public EndpointAddressesIterator { + public: + struct ClusterEntry { + std::string cluster_name; + std::shared_ptr endpoints; + std::vector priority_child_numbers; + + ClusterEntry(std::string cluster, + std::shared_ptr resource, + std::vector child_numbers) + : cluster_name(std::move(cluster)), + endpoints(std::move(resource)), + priority_child_numbers(std::move(child_numbers)) {} + + std::string GetChildPolicyName(size_t priority) const { + return MakeChildPolicyName(cluster_name, + priority_child_numbers[priority]); + } + }; + + explicit PriorityEndpointIterator(std::vector results) + : results_(std::move(results)) {} + + void ForEach(absl::FunctionRef callback) + const override { + for (const auto& entry : results_) { + const auto& priority_list = GetUpdatePriorityList(entry.endpoints.get()); + for (size_t priority = 0; priority < priority_list.size(); ++priority) { + const auto& priority_entry = priority_list[priority]; + std::string priority_child_name = entry.GetChildPolicyName(priority); + for (const auto& p : priority_entry.localities) { + const auto& locality_name = p.first; + const auto& locality = p.second; + std::vector hierarchical_path = { + RefCountedStringValue(priority_child_name), + RefCountedStringValue(locality_name->AsHumanReadableString())}; + auto hierarchical_path_attr = + MakeRefCounted(std::move(hierarchical_path)); + for (const auto& endpoint : locality.endpoints) { + uint32_t endpoint_weight = + locality.lb_weight * + endpoint.args().GetInt(GRPC_ARG_ADDRESS_WEIGHT).value_or(1); + callback(EndpointAddresses( + endpoint.addresses(), + endpoint.args() + .SetObject(hierarchical_path_attr) + .Set(GRPC_ARG_ADDRESS_WEIGHT, endpoint_weight) + .SetObject(locality_name->Ref()) + .Set(GRPC_ARG_XDS_LOCALITY_WEIGHT, locality.lb_weight))); + } + } + } } } - root_certificate_provider_ = std::move(new_root_provider); - xds_certificate_provider_->UpdateRootCertNameAndDistributor( - cluster_name, root_provider_cert_name, - root_certificate_provider_ == nullptr - ? nullptr - : root_certificate_provider_->distributor()); - // Configure identity cert. - absl::string_view identity_provider_instance_name = - cluster_data.common_tls_context.tls_certificate_provider_instance - .instance_name; - absl::string_view identity_provider_cert_name = - cluster_data.common_tls_context.tls_certificate_provider_instance - .certificate_name; - RefCountedPtr new_identity_provider; - if (!identity_provider_instance_name.empty()) { - new_identity_provider = - xds_client_->certificate_provider_store() - .CreateOrGetCertificateProvider(identity_provider_instance_name); - if (new_identity_provider == nullptr) { - return absl::UnavailableError( - absl::StrCat("Certificate provider instance name: \"", - identity_provider_instance_name, "\" not recognized.")); + + private: + std::vector results_; +}; + +std::shared_ptr CdsLb::CreateChildPolicyAddresses( + const std::vector& new_cluster_list) { + std::vector entries; + entries.reserve(new_cluster_list.size()); + size_t numbers_index = 0; + for (const auto* cluster_config : new_cluster_list) { + GPR_ASSERT(numbers_index < child_name_state_list_.size()); + const auto& endpoint_config = + absl::get( + cluster_config->children); + entries.emplace_back( + cluster_config->cluster_name, endpoint_config.endpoints, + child_name_state_list_[numbers_index++].priority_child_numbers); + } + return std::make_shared(std::move(entries)); +} + +std::string CdsLb::CreateChildPolicyResolutionNote( + const std::vector& new_cluster_list) { + std::vector resolution_notes; + for (const auto* cluster_config : new_cluster_list) { + const auto& endpoint_config = + absl::get( + cluster_config->children); + if (!endpoint_config.resolution_note.empty()) { + resolution_notes.push_back(endpoint_config.resolution_note); } } - identity_certificate_provider_ = std::move(new_identity_provider); - xds_certificate_provider_->UpdateIdentityCertNameAndDistributor( - cluster_name, identity_provider_cert_name, - identity_certificate_provider_ == nullptr - ? nullptr - : identity_certificate_provider_->distributor()); - // Configure SAN matchers. - const std::vector& match_subject_alt_names = - cluster_data.common_tls_context.certificate_validation_context - .match_subject_alt_names; - xds_certificate_provider_->UpdateSubjectAlternativeNameMatchers( - cluster_name, match_subject_alt_names); - return absl::OkStatus(); + return absl::StrJoin(resolution_notes, "; "); +} + +void CdsLb::ResetState() { + cluster_name_.clear(); + xds_config_.reset(); + child_name_state_list_.clear(); + if (child_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + interested_parties()); + child_policy_.reset(); + } } -void CdsLb::CancelClusterDataWatch(absl::string_view cluster_name, - ClusterWatcher* watcher, - bool delay_unsubscription) { - if (xds_certificate_provider_ != nullptr) { - std::string name(cluster_name); - xds_certificate_provider_->UpdateRootCertNameAndDistributor(name, "", - nullptr); - xds_certificate_provider_->UpdateIdentityCertNameAndDistributor(name, "", - nullptr); - xds_certificate_provider_->UpdateSubjectAlternativeNameMatchers(name, {}); +void CdsLb::ReportTransientFailure(absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { + gpr_log(GPR_INFO, "[cdslb %p] reporting TRANSIENT_FAILURE: %s", this, + status.ToString().c_str()); } - XdsClusterResourceType::CancelWatch(xds_client_.get(), cluster_name, watcher, - delay_unsubscription); + ResetState(); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, status, + MakeRefCounted(status)); } + // // factory // @@ -675,15 +754,7 @@ class CdsLbFactory : public LoadBalancingPolicyFactory { public: OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - auto xds_client = - args.args.GetObjectRef(DEBUG_LOCATION, "CdsLb"); - if (xds_client == nullptr) { - gpr_log(GPR_ERROR, - "XdsClient not present in channel args -- cannot instantiate " - "cds LB policy"); - return nullptr; - } - return MakeOrphanable(std::move(xds_client), std::move(args)); + return MakeOrphanable(std::move(args)); } absl::string_view name() const override { return kCds; } diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc index 9c477bb85b1..e2ff8824663 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc @@ -40,6 +40,7 @@ #include "src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h" #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_bootstrap_grpc.h" #include "src/core/ext/xds/xds_client.h" @@ -66,6 +67,7 @@ #include "src/core/lib/load_balancing/lb_policy_registry.h" #include "src/core/lib/load_balancing/subchannel_interface.h" #include "src/core/lib/resolver/endpoint_addresses.h" +#include "src/core/lib/security/credentials/xds/xds_credentials.h" #include "src/core/lib/transport/connectivity_state.h" namespace grpc_core { @@ -74,6 +76,8 @@ TraceFlag grpc_xds_cluster_impl_lb_trace(false, "xds_cluster_impl_lb"); namespace { +using XdsConfig = XdsDependencyManager::XdsConfig; + // // global circuit breaker atomic map // @@ -156,37 +160,24 @@ class XdsClusterImplLbConfig : public LoadBalancingPolicy::Config { absl::string_view name() const override { return kXdsClusterImpl; } + const std::string& cluster_name() const { return cluster_name_; } RefCountedPtr child_policy() const { return child_policy_; } - const std::string& cluster_name() const { return cluster_name_; } - const std::string& eds_service_name() const { return eds_service_name_; } - const absl::optional& - lrs_load_reporting_server() const { - return lrs_load_reporting_server_; - }; - uint32_t max_concurrent_requests() const { return max_concurrent_requests_; } - RefCountedPtr drop_config() const { - return drop_config_; - } static const JsonLoaderInterface* JsonLoader(const JsonArgs&); void JsonPostLoad(const Json& json, const JsonArgs& args, ValidationErrors* errors); private: - RefCountedPtr child_policy_; std::string cluster_name_; - std::string eds_service_name_; - absl::optional lrs_load_reporting_server_; - uint32_t max_concurrent_requests_; - RefCountedPtr drop_config_; + RefCountedPtr child_policy_; }; // xDS Cluster Impl LB policy. class XdsClusterImplLb : public LoadBalancingPolicy { public: - XdsClusterImplLb(RefCountedPtr xds_client, Args args); + XdsClusterImplLb(RefCountedPtr xds_client, Args args); absl::string_view name() const override { return kXdsClusterImpl; } @@ -247,16 +238,25 @@ class XdsClusterImplLb : public LoadBalancingPolicy { void ShutdownLocked() override; + void ResetState(); + void ReportTransientFailure(absl::Status status); + OrphanablePtr CreateChildPolicyLocked( const ChannelArgs& args); absl::Status UpdateChildPolicyLocked( absl::StatusOr> addresses, std::string resolution_note, const ChannelArgs& args); + absl::StatusOr> + MaybeCreateCertificateProviderLocked( + const XdsClusterResource& cluster_resource) const; + void MaybeUpdatePickerLocked(); // Current config from the resolver. RefCountedPtr config_; + std::shared_ptr cluster_resource_; + RefCountedPtr drop_config_; // Current concurrent number of requests. RefCountedPtr call_counter_; @@ -265,7 +265,7 @@ class XdsClusterImplLb : public LoadBalancingPolicy { bool shutting_down_ = false; // The xds client. - RefCountedPtr xds_client_; + RefCountedPtr xds_client_; // The stats for client-side load reporting. RefCountedPtr drop_stats_; @@ -357,8 +357,8 @@ XdsClusterImplLb::Picker::Picker(XdsClusterImplLb* xds_cluster_impl_lb, RefCountedPtr picker) : call_counter_(xds_cluster_impl_lb->call_counter_), max_concurrent_requests_( - xds_cluster_impl_lb->config_->max_concurrent_requests()), - drop_config_(xds_cluster_impl_lb->config_->drop_config()), + xds_cluster_impl_lb->cluster_resource_->max_concurrent_requests), + drop_config_(xds_cluster_impl_lb->drop_config_), drop_stats_(xds_cluster_impl_lb->drop_stats_), picker_(std::move(picker)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_cluster_impl_lb_trace)) { @@ -423,7 +423,7 @@ LoadBalancingPolicy::PickResult XdsClusterImplLb::Picker::Pick( // XdsClusterImplLb // -XdsClusterImplLb::XdsClusterImplLb(RefCountedPtr xds_client, +XdsClusterImplLb::XdsClusterImplLb(RefCountedPtr xds_client, Args args) : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_cluster_impl_lb_trace)) { @@ -445,6 +445,11 @@ void XdsClusterImplLb::ShutdownLocked() { gpr_log(GPR_INFO, "[xds_cluster_impl_lb %p] shutting down", this); } shutting_down_ = true; + ResetState(); + xds_client_.reset(DEBUG_LOCATION, "XdsClusterImpl"); +} + +void XdsClusterImplLb::ResetState() { // Remove the child policy's interested_parties pollset_set from the // xDS policy. if (child_policy_ != nullptr) { @@ -456,7 +461,18 @@ void XdsClusterImplLb::ShutdownLocked() { // the child. picker_.reset(); drop_stats_.reset(); - xds_client_.reset(DEBUG_LOCATION, "XdsClusterImpl"); +} + +void XdsClusterImplLb::ReportTransientFailure(absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_cluster_impl_lb_trace)) { + gpr_log(GPR_INFO, + "[xds_cluster_impl_lb %p] reporting TRANSIENT_FAILURE: %s", this, + status.ToString().c_str()); + } + ResetState(); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, status, + MakeRefCounted(status)); } void XdsClusterImplLb::ExitIdleLocked() { @@ -469,56 +485,174 @@ void XdsClusterImplLb::ResetBackoffLocked() { if (child_policy_ != nullptr) child_policy_->ResetBackoffLocked(); } +std::string GetEdsResourceName(const XdsClusterResource& cluster_resource) { + auto* eds = absl::get_if(&cluster_resource.type); + if (eds == nullptr) return ""; + return eds->eds_service_name; +} + absl::Status XdsClusterImplLb::UpdateLocked(UpdateArgs args) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_cluster_impl_lb_trace)) { gpr_log(GPR_INFO, "[xds_cluster_impl_lb %p] Received update", this); } - // Update config. - const bool is_initial_update = config_ == nullptr; - auto old_config = std::move(config_); - config_ = args.config.TakeAsSubclass(); - // On initial update, create drop stats. - if (is_initial_update) { - if (config_->lrs_load_reporting_server().has_value()) { - drop_stats_ = xds_client_->AddClusterDropStats( - config_->lrs_load_reporting_server().value(), config_->cluster_name(), - config_->eds_service_name()); - if (drop_stats_ == nullptr) { - gpr_log(GPR_ERROR, - "[xds_cluster_impl_lb %p] Failed to get cluster drop stats for " - "LRS server %s, cluster %s, EDS service name %s, load " - "reporting for drops will not be done.", - this, - config_->lrs_load_reporting_server()->server_uri().c_str(), - config_->cluster_name().c_str(), - config_->eds_service_name().c_str()); - } + // Grab new LB policy config. + auto new_config = args.config.TakeAsSubclass(); + // Cluster name should never change, because the cds policy will assign a + // different priority child name if that happens, which means that this + // policy instance will get replaced instead of being updated. + if (config_ != nullptr) { + GPR_ASSERT(config_->cluster_name() == new_config->cluster_name()); + } + // Get xDS config. + auto new_xds_config = args.args.GetObjectRef(); + if (new_xds_config == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError( + "xDS config not passed to xds_cluster_impl LB policy"); + ReportTransientFailure(status); + return status; + } + auto it = new_xds_config->clusters.find(new_config->cluster_name()); + if (it == new_xds_config->clusters.end() || !it->second.ok() || + it->second->cluster == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError(absl::StrCat( + "xDS config has no entry for cluster ", new_config->cluster_name())); + ReportTransientFailure(status); + return status; + } + auto& new_cluster_config = *it->second; + auto* endpoint_config = + absl::get_if( + &new_cluster_config.children); + if (endpoint_config == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError( + absl::StrCat("cluster config for ", new_config->cluster_name(), + " has no endpoint config")); + ReportTransientFailure(status); + return status; + } + auto xds_cert_provider = + MaybeCreateCertificateProviderLocked(*new_cluster_config.cluster); + if (!xds_cert_provider.ok()) { + // Should never happen. + ReportTransientFailure(xds_cert_provider.status()); + return xds_cert_provider.status(); + } + if (*xds_cert_provider != nullptr) { + args.args = args.args.SetObject(std::move(*xds_cert_provider)); + } + // Now we've verified the new config is good. + // Get new and old (if any) EDS service name. + std::string new_eds_service_name = + GetEdsResourceName(*new_cluster_config.cluster); + std::string old_eds_service_name = + cluster_resource_ == nullptr ? "" + : GetEdsResourceName(*cluster_resource_); + // Update drop stats if needed. + // Note: We need a drop stats object whenever load reporting is enabled, + // even if we have no EDS drop config, because we also use it when + // reporting circuit breaker drops. + if (!new_cluster_config.cluster->lrs_load_reporting_server.has_value()) { + drop_stats_.reset(); + } else if (cluster_resource_ == nullptr || + old_eds_service_name != new_eds_service_name || + cluster_resource_->lrs_load_reporting_server != + new_cluster_config.cluster->lrs_load_reporting_server) { + drop_stats_ = xds_client_->AddClusterDropStats( + *new_cluster_config.cluster->lrs_load_reporting_server, + new_config->cluster_name(), new_eds_service_name); + if (drop_stats_ == nullptr) { + gpr_log( + GPR_ERROR, + "[xds_cluster_impl_lb %p] Failed to get cluster drop stats for " + "LRS server %s, cluster %s, EDS service name %s, load " + "reporting for drops will not be done.", + this, + new_cluster_config.cluster->lrs_load_reporting_server->server_uri() + .c_str(), + new_config->cluster_name().c_str(), new_eds_service_name.c_str()); } - call_counter_ = g_call_counter_map->GetOrCreate( - config_->cluster_name(), config_->eds_service_name()); - } else { - // Cluster name, EDS service name, and LRS server name should never - // change, because the xds_cluster_resolver policy above us should be - // swapped out if that happens. - GPR_ASSERT(config_->cluster_name() == old_config->cluster_name()); - GPR_ASSERT(config_->eds_service_name() == old_config->eds_service_name()); - GPR_ASSERT(config_->lrs_load_reporting_server() == - old_config->lrs_load_reporting_server()); - } - // Update picker if max_concurrent_requests has changed. - if (is_initial_update || config_->max_concurrent_requests() != - old_config->max_concurrent_requests()) { - MaybeUpdatePickerLocked(); } + // Update call counter if needed. + if (cluster_resource_ == nullptr || + old_eds_service_name != new_eds_service_name) { + call_counter_ = g_call_counter_map->GetOrCreate(new_config->cluster_name(), + new_eds_service_name); + } + // Update config state, now that we're done comparing old and new fields. + config_ = std::move(new_config); + cluster_resource_ = new_cluster_config.cluster; + drop_config_ = endpoint_config->endpoints != nullptr + ? endpoint_config->endpoints->drop_config + : nullptr; + // Update picker in case some dependent config field changed. + MaybeUpdatePickerLocked(); // Update child policy. return UpdateChildPolicyLocked(std::move(args.addresses), std::move(args.resolution_note), args.args); } +absl::StatusOr> +XdsClusterImplLb::MaybeCreateCertificateProviderLocked( + const XdsClusterResource& cluster_resource) const { + // If the channel is not using XdsCreds, do nothing. + auto channel_credentials = channel_control_helper()->GetChannelCredentials(); + if (channel_credentials == nullptr || + channel_credentials->type() != XdsCredentials::Type()) { + return nullptr; + } + // Configure root cert. + absl::string_view root_provider_instance_name = + cluster_resource.common_tls_context.certificate_validation_context + .ca_certificate_provider_instance.instance_name; + absl::string_view root_cert_name = + cluster_resource.common_tls_context.certificate_validation_context + .ca_certificate_provider_instance.certificate_name; + RefCountedPtr root_cert_provider; + if (!root_provider_instance_name.empty()) { + root_cert_provider = + xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider(root_provider_instance_name); + if (root_cert_provider == nullptr) { + return absl::InternalError( + absl::StrCat("Certificate provider instance name: \"", + root_provider_instance_name, "\" not recognized.")); + } + } + // Configure identity cert. + absl::string_view identity_provider_instance_name = + cluster_resource.common_tls_context.tls_certificate_provider_instance + .instance_name; + absl::string_view identity_cert_name = + cluster_resource.common_tls_context.tls_certificate_provider_instance + .certificate_name; + RefCountedPtr identity_cert_provider; + if (!identity_provider_instance_name.empty()) { + identity_cert_provider = + xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider(identity_provider_instance_name); + if (identity_cert_provider == nullptr) { + return absl::InternalError( + absl::StrCat("Certificate provider instance name: \"", + identity_provider_instance_name, "\" not recognized.")); + } + } + // Configure SAN matchers. + const std::vector& san_matchers = + cluster_resource.common_tls_context.certificate_validation_context + .match_subject_alt_names; + // Create xds cert provider. + return MakeRefCounted( + root_cert_provider, root_cert_name, identity_cert_provider, + identity_cert_name, san_matchers); +} + void XdsClusterImplLb::MaybeUpdatePickerLocked() { // If we're dropping all calls, report READY, regardless of what (or // whether) the child has reported. - if (config_->drop_config() != nullptr && config_->drop_config()->drop_all()) { + if (drop_config_ != nullptr && drop_config_->drop_all()) { auto drop_picker = MakeRefCounted(this, picker_); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_cluster_impl_lb_trace)) { gpr_log(GPR_INFO, @@ -601,28 +735,30 @@ RefCountedPtr XdsClusterImplLb::Helper::CreateSubchannel( if (parent()->shutting_down_) return nullptr; // If load reporting is enabled, wrap the subchannel such that it // includes the locality stats object, which will be used by the Picker. - if (parent()->config_->lrs_load_reporting_server().has_value()) { + if (parent()->cluster_resource_->lrs_load_reporting_server.has_value()) { auto locality_name = per_address_args.GetObjectRef(); RefCountedPtr locality_stats = parent()->xds_client_->AddClusterLocalityStats( - parent()->config_->lrs_load_reporting_server().value(), + parent()->cluster_resource_->lrs_load_reporting_server.value(), parent()->config_->cluster_name(), - parent()->config_->eds_service_name(), std::move(locality_name)); + GetEdsResourceName(*parent()->cluster_resource_), + std::move(locality_name)); if (locality_stats != nullptr) { return MakeRefCounted( parent()->channel_control_helper()->CreateSubchannel( address, per_address_args, args), std::move(locality_stats)); } - gpr_log( - GPR_ERROR, - "[xds_cluster_impl_lb %p] Failed to get locality stats object for " - "LRS server %s, cluster %s, EDS service name %s; load reports will " - "not be generated (not wrapping subchannel)", - parent(), - parent()->config_->lrs_load_reporting_server()->server_uri().c_str(), - parent()->config_->cluster_name().c_str(), - parent()->config_->eds_service_name().c_str()); + gpr_log(GPR_ERROR, + "[xds_cluster_impl_lb %p] Failed to get locality stats object for " + "LRS server %s, cluster %s, EDS service name %s; load reports will " + "not be generated (not wrapping subchannel)", + parent(), + parent() + ->cluster_resource_->lrs_load_reporting_server->server_uri() + .c_str(), + parent()->config_->cluster_name().c_str(), + GetEdsResourceName(*parent()->cluster_resource_).c_str()); } // Load reporting not enabled, so don't wrap the subchannel. return parent()->channel_control_helper()->CreateSubchannel( @@ -652,67 +788,31 @@ void XdsClusterImplLb::Helper::UpdateState( // factory // -struct DropCategory { - std::string category; - uint32_t requests_per_million; - - static const JsonLoaderInterface* JsonLoader(const JsonArgs&) { - static const auto* loader = - JsonObjectLoader() - .Field("category", &DropCategory::category) - .Field("requests_per_million", &DropCategory::requests_per_million) - .Finish(); - return loader; - } -}; - const JsonLoaderInterface* XdsClusterImplLbConfig::JsonLoader(const JsonArgs&) { static const auto* loader = JsonObjectLoader() // Note: Some fields require custom processing, so they are // handled in JsonPostLoad() instead. .Field("clusterName", &XdsClusterImplLbConfig::cluster_name_) - .OptionalField("edsServiceName", - &XdsClusterImplLbConfig::eds_service_name_) - .OptionalField("lrsLoadReportingServer", - &XdsClusterImplLbConfig::lrs_load_reporting_server_) - .OptionalField("maxConcurrentRequests", - &XdsClusterImplLbConfig::max_concurrent_requests_) .Finish(); return loader; } -void XdsClusterImplLbConfig::JsonPostLoad(const Json& json, - const JsonArgs& args, +void XdsClusterImplLbConfig::JsonPostLoad(const Json& json, const JsonArgs&, ValidationErrors* errors) { // Parse "childPolicy" field. - { - ValidationErrors::ScopedField field(errors, ".childPolicy"); - auto it = json.object().find("childPolicy"); - if (it == json.object().end()) { - errors->AddError("field not present"); + ValidationErrors::ScopedField field(errors, ".childPolicy"); + auto it = json.object().find("childPolicy"); + if (it == json.object().end()) { + errors->AddError("field not present"); + } else { + auto lb_config = + CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( + it->second); + if (!lb_config.ok()) { + errors->AddError(lb_config.status().message()); } else { - auto lb_config = CoreConfiguration::Get() - .lb_policy_registry() - .ParseLoadBalancingConfig(it->second); - if (!lb_config.ok()) { - errors->AddError(lb_config.status().message()); - } else { - child_policy_ = std::move(*lb_config); - } - } - } - // Parse "dropCategories" field. - { - auto value = LoadJsonObjectField>( - json.object(), args, "dropCategories", errors, /*required=*/false); - if (value.has_value()) { - drop_config_ = MakeRefCounted(); - for (size_t i = 0; i < value->size(); ++i) { - DropCategory& drop_category = (*value)[i]; - drop_config_->AddCategory(std::move(drop_category.category), - drop_category.requests_per_million); - } + child_policy_ = std::move(*lb_config); } } } diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc index 46504c52d11..447a1c5abf9 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc @@ -40,7 +40,7 @@ #include "src/core/ext/filters/client_channel/client_channel_internal.h" #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" -#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/debug/trace.h" diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc deleted file mode 100644 index 7e29c17553c..00000000000 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +++ /dev/null @@ -1,1231 +0,0 @@ -// -// Copyright 2018 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/functional/function_ref.h" -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" - -#include -#include -#include -#include - -#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" -#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h" -#include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" -#include "src/core/ext/xds/xds_bootstrap.h" -#include "src/core/ext/xds/xds_bootstrap_grpc.h" -#include "src/core/ext/xds/xds_client.h" -#include "src/core/ext/xds/xds_client_grpc.h" -#include "src/core/ext/xds/xds_client_stats.h" -#include "src/core/ext/xds/xds_endpoint.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/debug/trace.h" -#include "src/core/lib/gprpp/debug_location.h" -#include "src/core/lib/gprpp/no_destruct.h" -#include "src/core/lib/gprpp/orphanable.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/gprpp/ref_counted_string.h" -#include "src/core/lib/gprpp/validation_errors.h" -#include "src/core/lib/gprpp/work_serializer.h" -#include "src/core/lib/iomgr/pollset_set.h" -#include "src/core/lib/iomgr/resolved_address.h" -#include "src/core/lib/json/json.h" -#include "src/core/lib/json/json_args.h" -#include "src/core/lib/json/json_object_loader.h" -#include "src/core/lib/json/json_writer.h" -#include "src/core/lib/load_balancing/delegating_helper.h" -#include "src/core/lib/load_balancing/lb_policy.h" -#include "src/core/lib/load_balancing/lb_policy_factory.h" -#include "src/core/lib/load_balancing/lb_policy_registry.h" -#include "src/core/lib/resolver/endpoint_addresses.h" -#include "src/core/lib/resolver/resolver.h" -#include "src/core/lib/resolver/resolver_registry.h" - -#define GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT 10000 - -namespace grpc_core { - -TraceFlag grpc_lb_xds_cluster_resolver_trace(false, "xds_cluster_resolver_lb"); - -namespace { - -constexpr absl::string_view kXdsClusterResolver = - "xds_cluster_resolver_experimental"; - -// Config for EDS LB policy. -class XdsClusterResolverLbConfig : public LoadBalancingPolicy::Config { - public: - struct DiscoveryMechanism { - std::string cluster_name; - absl::optional lrs_load_reporting_server; - uint32_t max_concurrent_requests; - enum DiscoveryMechanismType { - EDS, - LOGICAL_DNS, - }; - DiscoveryMechanismType type; - std::string eds_service_name; - std::string dns_hostname; - - Json::Array override_host_statuses; - - // This is type Json::Object instead of OutlierDetectionConfig, because we - // don't actually need to validate the contents of the outlier detection - // config here. In this case, the JSON is generated by the CDS policy - // instead of coming from service config, so it's not actually any better - // to catch the problem here than it is to catch it in the - // outlier_detection policy itself, so here we just act as a pass-through. - absl::optional outlier_detection_lb_config; - - bool operator==(const DiscoveryMechanism& other) const { - return (cluster_name == other.cluster_name && - lrs_load_reporting_server == other.lrs_load_reporting_server && - max_concurrent_requests == other.max_concurrent_requests && - type == other.type && - eds_service_name == other.eds_service_name && - dns_hostname == other.dns_hostname && - override_host_statuses == other.override_host_statuses && - outlier_detection_lb_config == other.outlier_detection_lb_config); - } - - static const JsonLoaderInterface* JsonLoader(const JsonArgs&); - void JsonPostLoad(const Json& json, const JsonArgs& args, - ValidationErrors* errors); - }; - - XdsClusterResolverLbConfig() = default; - - XdsClusterResolverLbConfig(const XdsClusterResolverLbConfig&) = delete; - XdsClusterResolverLbConfig& operator=(const XdsClusterResolverLbConfig&) = - delete; - - XdsClusterResolverLbConfig(XdsClusterResolverLbConfig&& other) = delete; - XdsClusterResolverLbConfig& operator=(XdsClusterResolverLbConfig&& other) = - delete; - - absl::string_view name() const override { return kXdsClusterResolver; } - - const std::vector& discovery_mechanisms() const { - return discovery_mechanisms_; - } - - const Json& xds_lb_policy() const { return xds_lb_policy_; } - - static const JsonLoaderInterface* JsonLoader(const JsonArgs&); - void JsonPostLoad(const Json& json, const JsonArgs& args, - ValidationErrors* errors); - - private: - std::vector discovery_mechanisms_; - Json xds_lb_policy_; -}; - -// Xds Cluster Resolver LB policy. -class XdsClusterResolverLb : public LoadBalancingPolicy { - public: - XdsClusterResolverLb(RefCountedPtr xds_client, Args args); - - absl::string_view name() const override { return kXdsClusterResolver; } - - absl::Status UpdateLocked(UpdateArgs args) override; - void ResetBackoffLocked() override; - void ExitIdleLocked() override; - - private: - // Discovery Mechanism Base class - // - // Implemented by EDS and LOGICAL_DNS. - // - // Implementations are responsible for calling the LB policy's - // OnEndpointChanged(), OnError(), and OnResourceDoesNotExist() - // methods when the corresponding events occur. - // - // Must implement Orphan() method to cancel the watchers. - class DiscoveryMechanism : public InternallyRefCounted { - public: - DiscoveryMechanism( - RefCountedPtr xds_cluster_resolver_lb, - size_t index) - : parent_(std::move(xds_cluster_resolver_lb)), index_(index) {} - - XdsClusterResolverLb* parent() const { return parent_.get(); } - size_t index() const { return index_; } - - virtual void Start() = 0; - virtual Json::Array override_child_policy() = 0; - virtual bool disable_reresolution() = 0; - - private: - RefCountedPtr parent_; - // Stores its own index in the vector of DiscoveryMechanism. - size_t index_; - }; - - class EdsDiscoveryMechanism : public DiscoveryMechanism { - public: - EdsDiscoveryMechanism( - RefCountedPtr xds_cluster_resolver_lb, - size_t index) - : DiscoveryMechanism(std::move(xds_cluster_resolver_lb), index) {} - void Start() override; - void Orphan() override; - Json::Array override_child_policy() override { return Json::Array{}; } - bool disable_reresolution() override { return true; } - - private: - class EndpointWatcher : public XdsEndpointResourceType::WatcherInterface { - public: - explicit EndpointWatcher( - RefCountedPtr discovery_mechanism) - : discovery_mechanism_(std::move(discovery_mechanism)) {} - ~EndpointWatcher() override { - discovery_mechanism_.reset(DEBUG_LOCATION, "EndpointWatcher"); - } - void OnResourceChanged(std::shared_ptr update, - RefCountedPtr - read_delay_handle) override { - discovery_mechanism_->parent()->work_serializer()->Run( - [self = RefAsSubclass(), - update = std::move(update), - read_delay_handle = std::move(read_delay_handle)]() mutable { - self->OnResourceChangedHelper(std::move(update)); - }, - DEBUG_LOCATION); - } - void OnError(absl::Status status, - RefCountedPtr read_delay_handle) - override { - discovery_mechanism_->parent()->work_serializer()->Run( - [self = RefAsSubclass(), - status = std::move(status), - read_delay_handle = std::move(read_delay_handle)]() mutable { - self->OnErrorHelper(std::move(status)); - }, - DEBUG_LOCATION); - } - void OnResourceDoesNotExist(RefCountedPtr - read_delay_handle) override { - discovery_mechanism_->parent()->work_serializer()->Run( - [self = RefAsSubclass(), - read_delay_handle = std::move(read_delay_handle)]() { - self->OnResourceDoesNotExistHelper(); - }, - DEBUG_LOCATION); - } - - private: - // Code accessing protected methods of `DiscoveryMechanism` need to be - // in methods of this class rather than in lambdas to work around an MSVC - // bug. - void OnResourceChangedHelper( - std::shared_ptr update) { - std::string resolution_note; - if (update->priorities.empty()) { - resolution_note = absl::StrCat( - "EDS resource ", discovery_mechanism_->GetEdsResourceName(), - " contains no localities"); - } else { - std::set empty_localities; - for (const auto& priority : update->priorities) { - for (const auto& p : priority.localities) { - if (p.second.endpoints.empty()) { - empty_localities.insert(p.first->AsHumanReadableString()); - } - } - } - if (!empty_localities.empty()) { - resolution_note = absl::StrCat( - "EDS resource ", discovery_mechanism_->GetEdsResourceName(), - " contains empty localities: [", - absl::StrJoin(empty_localities, "; "), "]"); - } - } - discovery_mechanism_->parent()->OnEndpointChanged( - discovery_mechanism_->index(), std::move(update), - std::move(resolution_note)); - } - void OnErrorHelper(absl::Status status) { - discovery_mechanism_->parent()->OnError( - discovery_mechanism_->index(), - absl::StrCat("EDS watcher error for resource ", - discovery_mechanism_->GetEdsResourceName(), " (", - status.ToString(), ")")); - } - void OnResourceDoesNotExistHelper() { - discovery_mechanism_->parent()->OnResourceDoesNotExist( - discovery_mechanism_->index(), - absl::StrCat("EDS resource ", - discovery_mechanism_->GetEdsResourceName(), - " does not exist")); - } - RefCountedPtr discovery_mechanism_; - }; - - // This is necessary only because of a bug in msvc where nested class - // cannot access protected member in base class. - friend class EndpointWatcher; - - absl::string_view GetEdsResourceName() const { - auto& config = parent()->config_->discovery_mechanisms()[index()]; - if (!config.eds_service_name.empty()) return config.eds_service_name; - return config.cluster_name; - } - - // Note that this is not owned, so this pointer must never be dereferenced. - EndpointWatcher* watcher_ = nullptr; - }; - - class LogicalDNSDiscoveryMechanism : public DiscoveryMechanism { - public: - LogicalDNSDiscoveryMechanism( - RefCountedPtr xds_cluster_resolver_lb, - size_t index) - : DiscoveryMechanism(std::move(xds_cluster_resolver_lb), index) {} - void Start() override; - void Orphan() override; - Json::Array override_child_policy() override { - return { - Json::FromObject({ - {"pick_first", Json::FromObject({})}, - }), - }; - } - bool disable_reresolution() override { return false; }; - - private: - class ResolverResultHandler : public Resolver::ResultHandler { - public: - explicit ResolverResultHandler( - RefCountedPtr discovery_mechanism) - : discovery_mechanism_(std::move(discovery_mechanism)) {} - - ~ResolverResultHandler() override {} - - void ReportResult(Resolver::Result result) override; - - private: - RefCountedPtr discovery_mechanism_; - }; - - // This is necessary only because of a bug in msvc where nested class cannot - // access protected member in base class. - friend class ResolverResultHandler; - - absl::string_view GetDnsHostname() const { - auto& config = parent()->config_->discovery_mechanisms()[index()]; - return config.dns_hostname; - } - - OrphanablePtr resolver_; - }; - - struct DiscoveryMechanismEntry { - OrphanablePtr discovery_mechanism; - // Most recent update reported by the discovery mechanism. - std::shared_ptr latest_update; - // Last resolution note reported by the discovery mechanism, if any. - std::string resolution_note; - // State used to retain child policy names for priority policy. - std::vector priority_child_numbers; - size_t next_available_child_number = 0; - - const XdsClusterResolverLbConfig::DiscoveryMechanism& config() const; - - // Returns the child policy name for a given priority. - std::string GetChildPolicyName(size_t priority) const; - }; - - class Helper : public ParentOwningDelegatingChannelControlHelper< - XdsClusterResolverLb> { - public: - explicit Helper( - RefCountedPtr xds_cluster_resolver_policy) - : ParentOwningDelegatingChannelControlHelper( - std::move(xds_cluster_resolver_policy)) {} - - // This is a no-op, because we get the addresses from the xds - // client, which is a watch-based API. - // TODO(roth): Don't we need to propagate this for LOGICAL_DNS clusters? - void RequestReresolution() override {} - }; - - ~XdsClusterResolverLb() override; - - void ShutdownLocked() override; - - void OnEndpointChanged(size_t index, - std::shared_ptr update, - std::string resolution_note); - void OnError(size_t index, std::string resolution_note); - void OnResourceDoesNotExist(size_t index, std::string resolution_note); - - void MaybeDestroyChildPolicyLocked(); - - absl::Status UpdateChildPolicyLocked(); - OrphanablePtr CreateChildPolicyLocked( - const ChannelArgs& args); - std::shared_ptr CreateChildPolicyAddressesLocked(); - std::string CreateChildPolicyResolutionNoteLocked(); - RefCountedPtr CreateChildPolicyConfigLocked(); - ChannelArgs CreateChildPolicyArgsLocked(const ChannelArgs& args_in); - - // The xds client and endpoint watcher. - RefCountedPtr xds_client_; - - // Current channel args and config from the resolver. - ChannelArgs args_; - RefCountedPtr config_; - - // Internal state. - bool shutting_down_ = false; - - // Vector of discovery mechansism entries in priority order. - std::vector discovery_mechanisms_; - - OrphanablePtr child_policy_; -}; - -// -// XdsClusterResolverLb::EdsDiscoveryMechanism -// - -void XdsClusterResolverLb::EdsDiscoveryMechanism::Start() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p] eds discovery mechanism %" PRIuPTR - ":%p starting xds watch for %s", - parent(), index(), this, std::string(GetEdsResourceName()).c_str()); - } - auto watcher = - MakeRefCounted(RefAsSubclass( - DEBUG_LOCATION, "EdsDiscoveryMechanism")); - watcher_ = watcher.get(); - XdsEndpointResourceType::StartWatch(parent()->xds_client_.get(), - GetEdsResourceName(), std::move(watcher)); -} - -void XdsClusterResolverLb::EdsDiscoveryMechanism::Orphan() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p] eds discovery mechanism %" PRIuPTR - ":%p cancelling xds watch for %s", - parent(), index(), this, std::string(GetEdsResourceName()).c_str()); - } - XdsEndpointResourceType::CancelWatch(parent()->xds_client_.get(), - GetEdsResourceName(), watcher_); - Unref(); -} - -// -// XdsClusterResolverLb::LogicalDNSDiscoveryMechanism -// - -void XdsClusterResolverLb::LogicalDNSDiscoveryMechanism::Start() { - std::string target; - ChannelArgs args = parent()->args_; - auto* fake_resolver_response_generator = - args.GetPointer( - GRPC_ARG_XDS_LOGICAL_DNS_CLUSTER_FAKE_RESOLVER_RESPONSE_GENERATOR); - if (fake_resolver_response_generator != nullptr) { - target = absl::StrCat("fake:", GetDnsHostname()); - args = args.SetObject(fake_resolver_response_generator->Ref()); - } else { - target = absl::StrCat("dns:", GetDnsHostname()); - } - resolver_ = CoreConfiguration::Get().resolver_registry().CreateResolver( - target.c_str(), args, parent()->interested_parties(), - parent()->work_serializer(), - std::make_unique( - RefAsSubclass( - DEBUG_LOCATION, "LogicalDNSDiscoveryMechanism"))); - if (resolver_ == nullptr) { - parent()->OnResourceDoesNotExist( - index(), - absl::StrCat("error creating DNS resolver for ", GetDnsHostname())); - return; - } - resolver_->StartLocked(); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p] logical DNS discovery mechanism " - "%" PRIuPTR ":%p starting dns resolver %p", - parent(), index(), this, resolver_.get()); - } -} - -void XdsClusterResolverLb::LogicalDNSDiscoveryMechanism::Orphan() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log( - GPR_INFO, - "[xds_cluster_resolver_lb %p] logical DNS discovery mechanism %" PRIuPTR - ":%p shutting down dns resolver %p", - parent(), index(), this, resolver_.get()); - } - resolver_.reset(); - Unref(); -} - -// -// XdsClusterResolverLb::LogicalDNSDiscoveryMechanism::ResolverResultHandler -// - -void XdsClusterResolverLb::LogicalDNSDiscoveryMechanism::ResolverResultHandler:: - ReportResult(Resolver::Result result) { - XdsClusterResolverLb* lb_policy = discovery_mechanism_->parent(); - size_t index = discovery_mechanism_->index(); - if (!result.addresses.ok()) { - if (result.resolution_note.empty()) { - result.resolution_note = absl::StrCat( - "DNS resolution failed for ", discovery_mechanism_->GetDnsHostname(), - " (", result.addresses.status().ToString(), ")"); - } - lb_policy->OnError(index, result.resolution_note); - return; - } - // Convert resolver result to EDS update. - auto update = std::make_shared(); - XdsEndpointResource::Priority::Locality locality; - locality.name = MakeRefCounted("", "", ""); - locality.lb_weight = 1; - locality.endpoints = std::move(*result.addresses); - XdsEndpointResource::Priority priority; - priority.localities.emplace(locality.name.get(), std::move(locality)); - update->priorities.emplace_back(std::move(priority)); - lb_policy->OnEndpointChanged(index, std::move(update), - std::move(result.resolution_note)); -} - -// -// XdsClusterResolverLb::DiscoveryMechanismEntry -// - -const XdsClusterResolverLbConfig::DiscoveryMechanism& -XdsClusterResolverLb::DiscoveryMechanismEntry::config() const { - return discovery_mechanism->parent() - ->config_->discovery_mechanisms()[discovery_mechanism->index()]; -} - -std::string MakeChildPolicyName(absl::string_view cluster_name, - size_t child_number) { - return absl::StrCat("{cluster=", cluster_name, - ", child_number=", child_number, "}"); -} - -std::string XdsClusterResolverLb::DiscoveryMechanismEntry::GetChildPolicyName( - size_t priority) const { - return MakeChildPolicyName(config().cluster_name, - priority_child_numbers[priority]); -} - -// -// XdsClusterResolverLb public methods -// - -XdsClusterResolverLb::XdsClusterResolverLb(RefCountedPtr xds_client, - Args args) - : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_cluster_resolver_lb %p] created -- xds_client=%p", - this, xds_client_.get()); - } -} - -XdsClusterResolverLb::~XdsClusterResolverLb() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p] destroying xds_cluster_resolver LB " - "policy", - this); - } -} - -void XdsClusterResolverLb::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_cluster_resolver_lb %p] shutting down", this); - } - shutting_down_ = true; - MaybeDestroyChildPolicyLocked(); - discovery_mechanisms_.clear(); - xds_client_.reset(DEBUG_LOCATION, "XdsClusterResolverLb"); - args_ = ChannelArgs(); -} - -void XdsClusterResolverLb::MaybeDestroyChildPolicyLocked() { - if (child_policy_ != nullptr) { - grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), - interested_parties()); - child_policy_.reset(); - } -} - -absl::Status XdsClusterResolverLb::UpdateLocked(UpdateArgs args) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_cluster_resolver_lb %p] Received update", this); - } - const bool is_initial_update = args_ == ChannelArgs(); - // Update config. - auto old_config = std::move(config_); - config_ = args.config.TakeAsSubclass(); - // Update args. - args_ = std::move(args.args); - // Update child policy if needed. - absl::Status status; - if (child_policy_ != nullptr) status = UpdateChildPolicyLocked(); - // Create endpoint watcher if needed. - if (is_initial_update) { - for (const auto& config : config_->discovery_mechanisms()) { - DiscoveryMechanismEntry entry; - if (config.type == XdsClusterResolverLbConfig::DiscoveryMechanism:: - DiscoveryMechanismType::EDS) { - entry.discovery_mechanism = MakeOrphanable( - RefAsSubclass(DEBUG_LOCATION, - "EdsDiscoveryMechanism"), - discovery_mechanisms_.size()); - } else if (config.type == XdsClusterResolverLbConfig::DiscoveryMechanism:: - DiscoveryMechanismType::LOGICAL_DNS) { - entry.discovery_mechanism = - MakeOrphanable( - RefAsSubclass( - DEBUG_LOCATION, "LogicalDNSDiscoveryMechanism"), - discovery_mechanisms_.size()); - } else { - GPR_ASSERT(0); - } - discovery_mechanisms_.push_back(std::move(entry)); - } - // Call start() on all discovery mechanisms after creation. - for (const auto& discovery_mechanism : discovery_mechanisms_) { - discovery_mechanism.discovery_mechanism->Start(); - } - } - return status; -} - -void XdsClusterResolverLb::ResetBackoffLocked() { - if (child_policy_ != nullptr) { - child_policy_->ResetBackoffLocked(); - } -} - -void XdsClusterResolverLb::ExitIdleLocked() { - if (child_policy_ != nullptr) child_policy_->ExitIdleLocked(); -} - -// We need at least one priority for each discovery mechanism, just so that we -// have a child in which to create the xds_cluster_impl policy. This ensures -// that we properly handle the case of a discovery mechanism dropping 100% of -// calls, the OnError() case, and the OnResourceDoesNotExist() case. -const XdsEndpointResource::PriorityList& GetUpdatePriorityList( - const XdsEndpointResource& update) { - static const NoDestruct - kPriorityListWithEmptyPriority(1); - if (update.priorities.empty()) return *kPriorityListWithEmptyPriority; - return update.priorities; -} - -void XdsClusterResolverLb::OnEndpointChanged( - size_t index, std::shared_ptr update, - std::string resolution_note) { - if (shutting_down_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p] Received update from xds client" - " for discovery mechanism %" PRIuPTR " (resolution_note=\"%s\")", - this, index, resolution_note.c_str()); - } - DiscoveryMechanismEntry& discovery_entry = discovery_mechanisms_[index]; - const XdsEndpointResource::PriorityList& priority_list = - GetUpdatePriorityList(*update); - // Update priority_child_numbers, reusing old child numbers in an - // intelligent way to avoid unnecessary churn. - // First, build some maps from locality to child number and the reverse - // from the old data in the entry's update and priority_child_numbers. - std::map - locality_child_map; - std::map> - child_locality_map; - if (discovery_entry.latest_update != nullptr) { - const auto& prev_priority_list = - GetUpdatePriorityList(*discovery_entry.latest_update); - for (size_t priority = 0; priority < prev_priority_list.size(); - ++priority) { - size_t child_number = discovery_entry.priority_child_numbers[priority]; - const auto& localities = prev_priority_list[priority].localities; - for (const auto& p : localities) { - XdsLocalityName* locality_name = p.first; - locality_child_map[locality_name] = child_number; - child_locality_map[child_number].insert(locality_name); - } - } - } - // Construct new list of children. - std::vector priority_child_numbers; - for (size_t priority = 0; priority < priority_list.size(); ++priority) { - const auto& localities = priority_list[priority].localities; - absl::optional child_number; - // If one of the localities in this priority already existed, reuse its - // child number. - for (const auto& p : localities) { - XdsLocalityName* locality_name = p.first; - if (!child_number.has_value()) { - auto it = locality_child_map.find(locality_name); - if (it != locality_child_map.end()) { - child_number = it->second; - locality_child_map.erase(it); - // Remove localities that *used* to be in this child number, so - // that we don't incorrectly reuse this child number for a - // subsequent priority. - for (XdsLocalityName* old_locality : - child_locality_map[*child_number]) { - locality_child_map.erase(old_locality); - } - } - } else { - // Remove all localities that are now in this child number, so - // that we don't accidentally reuse this child number for a - // subsequent priority. - locality_child_map.erase(locality_name); - } - } - // If we didn't find an existing child number, assign a new one. - if (!child_number.has_value()) { - for (child_number = discovery_entry.next_available_child_number; - child_locality_map.find(*child_number) != child_locality_map.end(); - ++(*child_number)) { - } - discovery_entry.next_available_child_number = *child_number + 1; - // Add entry so we know that the child number is in use. - // (Don't need to add the list of localities, since we won't use them.) - child_locality_map[*child_number]; - } - priority_child_numbers.push_back(*child_number); - } - // Save update. - discovery_entry.latest_update = std::move(update); - discovery_entry.resolution_note = std::move(resolution_note); - discovery_entry.priority_child_numbers = std::move(priority_child_numbers); - // If any discovery mechanism has not received its first update, - // wait until that happens before creating the child policy. - // TODO(roth): If this becomes problematic in the future (e.g., a - // secondary discovery mechanism delaying us from starting up at all), - // we can consider some sort of optimization whereby we can create the - // priority policy with only a subset of its children. But we need to - // make sure not to get into a situation where the priority policy - // will put the channel into TRANSIENT_FAILURE instead of CONNECTING - // while we're still waiting for the other discovery mechanism(s). - for (DiscoveryMechanismEntry& mechanism : discovery_mechanisms_) { - if (mechanism.latest_update == nullptr) return; - } - // Update child policy. - // TODO(roth): If the child policy reports an error with the update, - // we need to propagate that error back to the resolver somehow. - (void)UpdateChildPolicyLocked(); -} - -void XdsClusterResolverLb::OnError(size_t index, std::string resolution_note) { - gpr_log(GPR_ERROR, - "[xds_cluster_resolver_lb %p] discovery mechanism %" PRIuPTR - " reported error: %s", - this, index, resolution_note.c_str()); - if (shutting_down_) return; - if (discovery_mechanisms_[index].latest_update == nullptr) { - // Call OnEndpointChanged() with an empty update just like - // OnResourceDoesNotExist(). - OnEndpointChanged(index, std::make_shared(), - std::move(resolution_note)); - } -} - -void XdsClusterResolverLb::OnResourceDoesNotExist(size_t index, - std::string resolution_note) { - gpr_log(GPR_ERROR, - "[xds_cluster_resolver_lb %p] discovery mechanism %" PRIuPTR - " resource does not exist: %s", - this, index, resolution_note.c_str()); - if (shutting_down_) return; - // Call OnEndpointChanged() with an empty update. - OnEndpointChanged(index, std::make_shared(), - std::move(resolution_note)); -} - -// -// child policy-related methods -// - -class PriorityEndpointIterator : public EndpointAddressesIterator { - public: - struct DiscoveryMechanismResult { - std::shared_ptr update; - std::string cluster_name; - std::vector priority_child_numbers; - - DiscoveryMechanismResult( - std::shared_ptr resource, - std::string cluster, std::vector child_numbers) - : update(std::move(resource)), - cluster_name(std::move(cluster)), - priority_child_numbers(std::move(child_numbers)) {} - - std::string GetChildPolicyName(size_t priority) const { - return MakeChildPolicyName(cluster_name, - priority_child_numbers[priority]); - } - }; - - explicit PriorityEndpointIterator( - std::vector results) - : results_(std::move(results)) {} - - void ForEach(absl::FunctionRef callback) - const override { - for (const auto& entry : results_) { - const auto& priority_list = GetUpdatePriorityList(*entry.update); - for (size_t priority = 0; priority < priority_list.size(); ++priority) { - const auto& priority_entry = priority_list[priority]; - std::string priority_child_name = entry.GetChildPolicyName(priority); - for (const auto& p : priority_entry.localities) { - const auto& locality_name = p.first; - const auto& locality = p.second; - std::vector hierarchical_path = { - RefCountedStringValue(priority_child_name), - RefCountedStringValue(locality_name->AsHumanReadableString())}; - auto hierarchical_path_attr = - MakeRefCounted(std::move(hierarchical_path)); - for (const auto& endpoint : locality.endpoints) { - uint32_t endpoint_weight = - locality.lb_weight * - endpoint.args().GetInt(GRPC_ARG_ADDRESS_WEIGHT).value_or(1); - callback(EndpointAddresses( - endpoint.addresses(), - endpoint.args() - .SetObject(hierarchical_path_attr) - .Set(GRPC_ARG_ADDRESS_WEIGHT, endpoint_weight) - .SetObject(locality_name->Ref()) - .Set(GRPC_ARG_XDS_LOCALITY_WEIGHT, locality.lb_weight))); - } - } - } - } - } - - private: - std::vector results_; -}; - -std::shared_ptr -XdsClusterResolverLb::CreateChildPolicyAddressesLocked() { - std::vector entries; - entries.reserve(discovery_mechanisms_.size()); - for (const auto& discovery_entry : discovery_mechanisms_) { - entries.emplace_back(discovery_entry.latest_update, - discovery_entry.config().cluster_name, - discovery_entry.priority_child_numbers); - } - return std::make_shared(std::move(entries)); -} - -std::string XdsClusterResolverLb::CreateChildPolicyResolutionNoteLocked() { - std::vector resolution_notes; - for (const auto& discovery_entry : discovery_mechanisms_) { - if (!discovery_entry.resolution_note.empty()) { - resolution_notes.push_back(discovery_entry.resolution_note); - } - } - return absl::StrJoin(resolution_notes, "; "); -} - -RefCountedPtr -XdsClusterResolverLb::CreateChildPolicyConfigLocked() { - Json::Object priority_children; - Json::Array priority_priorities; - for (const auto& discovery_entry : discovery_mechanisms_) { - const auto& priority_list = - GetUpdatePriorityList(*discovery_entry.latest_update); - const auto& discovery_config = discovery_entry.config(); - for (size_t priority = 0; priority < priority_list.size(); ++priority) { - // Determine what xDS LB policy to use. - Json child_policy; - if (!discovery_entry.discovery_mechanism->override_child_policy() - .empty()) { - child_policy = Json::FromArray( - discovery_entry.discovery_mechanism->override_child_policy()); - } else { - child_policy = config_->xds_lb_policy(); - } - // Wrap the xDS LB policy in the xds_override_host policy. - Json::Object xds_override_host_lb_config = { - {"childPolicy", std::move(child_policy)}, - }; - if (!discovery_config.override_host_statuses.empty()) { - xds_override_host_lb_config["overrideHostStatus"] = - Json::FromArray(discovery_config.override_host_statuses); - } - Json::Array xds_override_host_config = {Json::FromObject({ - {"xds_override_host_experimental", - Json::FromObject(std::move(xds_override_host_lb_config))}, - })}; - // Wrap it in the xds_cluster_impl policy. - Json::Object xds_cluster_impl_config = { - {"clusterName", Json::FromString(discovery_config.cluster_name)}, - {"childPolicy", Json::FromArray(std::move(xds_override_host_config))}, - {"maxConcurrentRequests", - Json::FromNumber(discovery_config.max_concurrent_requests)}, - }; - if (discovery_entry.latest_update->drop_config != nullptr) { - Json::Array drop_categories; - for (const auto& category : - discovery_entry.latest_update->drop_config->drop_category_list()) { - drop_categories.push_back(Json::FromObject({ - {"category", Json::FromString(category.name)}, - {"requests_per_million", - Json::FromNumber(category.parts_per_million)}, - })); - } - if (!drop_categories.empty()) { - xds_cluster_impl_config["dropCategories"] = - Json::FromArray(std::move(drop_categories)); - } - } - if (!discovery_config.eds_service_name.empty()) { - xds_cluster_impl_config["edsServiceName"] = - Json::FromString(discovery_config.eds_service_name); - } - if (discovery_config.lrs_load_reporting_server.has_value()) { - xds_cluster_impl_config["lrsLoadReportingServer"] = - discovery_config.lrs_load_reporting_server->ToJson(); - } - // Wrap it in the outlier_detection policy. - Json::Object outlier_detection_config; - if (discovery_entry.config().outlier_detection_lb_config.has_value()) { - outlier_detection_config = - discovery_entry.config().outlier_detection_lb_config.value(); - } - outlier_detection_config["childPolicy"] = - Json::FromArray({Json::FromObject({ - {"xds_cluster_impl_experimental", - Json::FromObject(std::move(xds_cluster_impl_config))}, - })}); - Json locality_picking_policy = Json::FromArray({Json::FromObject({ - {"outlier_detection_experimental", - Json::FromObject(std::move(outlier_detection_config))}, - })}); - // Add priority entry, with the appropriate child name. - std::string child_name = discovery_entry.GetChildPolicyName(priority); - priority_priorities.emplace_back(Json::FromString(child_name)); - Json::Object child_config = { - {"config", std::move(locality_picking_policy)}, - }; - if (discovery_entry.discovery_mechanism->disable_reresolution()) { - child_config["ignore_reresolution_requests"] = Json::FromBool(true); - } - priority_children[child_name] = Json::FromObject(std::move(child_config)); - } - } - Json json = Json::FromArray({Json::FromObject({ - {"priority_experimental", - Json::FromObject({ - {"children", Json::FromObject(std::move(priority_children))}, - {"priorities", Json::FromArray(std::move(priority_priorities))}, - })}, - })}); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log( - GPR_INFO, - "[xds_cluster_resolver_lb %p] generated config for child policy: %s", - this, JsonDump(json, /*indent=*/1).c_str()); - } - auto config = - CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( - json); - if (!config.ok()) { - // This should never happen, but if it does, we basically have no - // way to fix it, so we put the channel in TRANSIENT_FAILURE. - gpr_log(GPR_ERROR, - "[xds_cluster_resolver_lb %p] error parsing generated child policy " - "config -- " - "will put channel in TRANSIENT_FAILURE: %s", - this, config.status().ToString().c_str()); - absl::Status status = absl::InternalError( - "xds_cluster_resolver LB policy: error parsing generated child policy " - "config"); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, status, - MakeRefCounted(status)); - return nullptr; - } - return std::move(*config); -} - -absl::Status XdsClusterResolverLb::UpdateChildPolicyLocked() { - if (shutting_down_) return absl::OkStatus(); - UpdateArgs update_args; - update_args.config = CreateChildPolicyConfigLocked(); - if (update_args.config == nullptr) return absl::OkStatus(); - update_args.addresses = CreateChildPolicyAddressesLocked(); - update_args.resolution_note = CreateChildPolicyResolutionNoteLocked(); - update_args.args = CreateChildPolicyArgsLocked(args_); - if (child_policy_ == nullptr) { - child_policy_ = CreateChildPolicyLocked(update_args.args); - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_cluster_resolver_lb %p] Updating child policy %p", - this, child_policy_.get()); - } - return child_policy_->UpdateLocked(std::move(update_args)); -} - -ChannelArgs XdsClusterResolverLb::CreateChildPolicyArgsLocked( - const ChannelArgs& args) { - // Inhibit client-side health checking, since the balancer does this - // for us. - return args.Set(GRPC_ARG_INHIBIT_HEALTH_CHECKING, 1); -} - -OrphanablePtr -XdsClusterResolverLb::CreateChildPolicyLocked(const ChannelArgs& args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.work_serializer = work_serializer(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = std::make_unique( - RefAsSubclass(DEBUG_LOCATION, "Helper")); - OrphanablePtr lb_policy = - CoreConfiguration::Get().lb_policy_registry().CreateLoadBalancingPolicy( - "priority_experimental", std::move(lb_policy_args)); - if (GPR_UNLIKELY(lb_policy == nullptr)) { - gpr_log(GPR_ERROR, - "[xds_cluster_resolver_lb %p] failure creating child policy", this); - return nullptr; - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_cluster_resolver_trace)) { - gpr_log(GPR_INFO, - "[xds_cluster_resolver_lb %p]: Created new child policy %p", this, - lb_policy.get()); - } - // Add our interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on - // this policy, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - interested_parties()); - return lb_policy; -} - -// -// factory -// - -const JsonLoaderInterface* -XdsClusterResolverLbConfig::DiscoveryMechanism::JsonLoader(const JsonArgs&) { - static const auto* loader = - JsonObjectLoader() - // Note: Several fields requires custom processing, - // so they are handled in JsonPostLoad() instead. - .Field("clusterName", &DiscoveryMechanism::cluster_name) - .OptionalField("lrsLoadReportingServer", - &DiscoveryMechanism::lrs_load_reporting_server) - .OptionalField("max_concurrent_requests", - &DiscoveryMechanism::max_concurrent_requests) - .OptionalField("outlierDetection", - &DiscoveryMechanism::outlier_detection_lb_config) - .OptionalField("overrideHostStatus", - &DiscoveryMechanism::override_host_statuses) - .Finish(); - return loader; -} - -void XdsClusterResolverLbConfig::DiscoveryMechanism::JsonPostLoad( - const Json& json, const JsonArgs& args, ValidationErrors* errors) { - // Parse "type". - { - auto type_field = - LoadJsonObjectField(json.object(), args, "type", errors); - if (type_field.has_value()) { - if (*type_field == "EDS") { - type = DiscoveryMechanismType::EDS; - } else if (*type_field == "LOGICAL_DNS") { - type = DiscoveryMechanismType::LOGICAL_DNS; - } else { - ValidationErrors::ScopedField field(errors, ".type"); - errors->AddError(absl::StrCat("unknown type \"", *type_field, "\"")); - } - } - } - // Parse "edsServiceName" if type is EDS. - if (type == DiscoveryMechanismType::EDS) { - auto value = LoadJsonObjectField(json.object(), args, - "edsServiceName", errors, - /*required=*/false); - if (value.has_value()) eds_service_name = std::move(*value); - } - // Parse "dnsHostname" if type is LOGICAL_DNS. - if (type == DiscoveryMechanismType::LOGICAL_DNS) { - auto value = LoadJsonObjectField(json.object(), args, - "dnsHostname", errors); - if (value.has_value()) dns_hostname = std::move(*value); - } -} - -const JsonLoaderInterface* XdsClusterResolverLbConfig::JsonLoader( - const JsonArgs&) { - static const auto* loader = - JsonObjectLoader() - // Note: The "xdsLbPolicy" field requires custom processing, - // so it's handled in JsonPostLoad() instead. - .Field("discoveryMechanisms", - &XdsClusterResolverLbConfig::discovery_mechanisms_) - .Finish(); - return loader; -} - -void XdsClusterResolverLbConfig::JsonPostLoad(const Json& json, const JsonArgs&, - ValidationErrors* errors) { - // Validate discoveryMechanisms. - { - ValidationErrors::ScopedField field(errors, ".discoveryMechanisms"); - if (!errors->FieldHasErrors() && discovery_mechanisms_.empty()) { - errors->AddError("must be non-empty"); - } - } - // Parse "xdsLbPolicy". - { - ValidationErrors::ScopedField field(errors, ".xdsLbPolicy"); - auto it = json.object().find("xdsLbPolicy"); - if (it == json.object().end()) { - errors->AddError("field not present"); - } else { - auto lb_config = CoreConfiguration::Get() - .lb_policy_registry() - .ParseLoadBalancingConfig(it->second); - if (!lb_config.ok()) errors->AddError(lb_config.status().message()); - xds_lb_policy_ = it->second; - } - } -} - -class XdsClusterResolverLbFactory : public LoadBalancingPolicyFactory { - public: - OrphanablePtr CreateLoadBalancingPolicy( - LoadBalancingPolicy::Args args) const override { - auto xds_client = args.args.GetObjectRef( - DEBUG_LOCATION, "XdsClusterResolverLbFactory"); - if (xds_client == nullptr) { - gpr_log(GPR_ERROR, - "XdsClient not present in channel args -- cannot instantiate " - "xds_cluster_resolver LB policy"); - return nullptr; - } - return MakeOrphanable(std::move(xds_client), - std::move(args)); - } - - absl::string_view name() const override { return kXdsClusterResolver; } - - absl::StatusOr> - ParseLoadBalancingConfig(const Json& json) const override { - return LoadFromJson>( - json, JsonArgs(), - "errors validating xds_cluster_resolver LB policy config"); - } - - private: - class XdsClusterResolverChildHandler : public ChildPolicyHandler { - public: - XdsClusterResolverChildHandler(RefCountedPtr xds_client, - Args args) - : ChildPolicyHandler(std::move(args), - &grpc_lb_xds_cluster_resolver_trace), - xds_client_(std::move(xds_client)) {} - - ~XdsClusterResolverChildHandler() override { - xds_client_.reset(DEBUG_LOCATION, "XdsClusterResolverChildHandler"); - } - - bool ConfigChangeRequiresNewPolicyInstance( - LoadBalancingPolicy::Config* old_config, - LoadBalancingPolicy::Config* new_config) const override { - GPR_ASSERT(old_config->name() == kXdsClusterResolver); - GPR_ASSERT(new_config->name() == kXdsClusterResolver); - XdsClusterResolverLbConfig* old_xds_cluster_resolver_config = - static_cast(old_config); - XdsClusterResolverLbConfig* new_xds_cluster_resolver_config = - static_cast(new_config); - if (old_xds_cluster_resolver_config->discovery_mechanisms().size() != - new_xds_cluster_resolver_config->discovery_mechanisms().size()) { - return true; - } - for (size_t i = 0; - i < old_xds_cluster_resolver_config->discovery_mechanisms().size(); - ++i) { - auto& old_discovery_mechanism = - old_xds_cluster_resolver_config->discovery_mechanisms()[i]; - auto& new_discovery_mechanism = - new_xds_cluster_resolver_config->discovery_mechanisms()[i]; - if (old_discovery_mechanism.type != new_discovery_mechanism.type || - old_discovery_mechanism.cluster_name != - new_discovery_mechanism.cluster_name || - old_discovery_mechanism.eds_service_name != - new_discovery_mechanism.eds_service_name || - old_discovery_mechanism.dns_hostname != - new_discovery_mechanism.dns_hostname || - !(old_discovery_mechanism.lrs_load_reporting_server == - new_discovery_mechanism.lrs_load_reporting_server)) { - return true; - } - } - return false; - } - - OrphanablePtr CreateLoadBalancingPolicy( - absl::string_view /*name*/, - LoadBalancingPolicy::Args args) const override { - return MakeOrphanable( - xds_client_->Ref(DEBUG_LOCATION, "XdsClusterResolverLb"), - std::move(args)); - } - - private: - RefCountedPtr xds_client_; - }; -}; - -} // namespace - -void RegisterXdsClusterResolverLbPolicy(CoreConfiguration::Builder* builder) { - builder->lb_policy_registry()->RegisterLoadBalancingPolicyFactory( - std::make_unique()); -} - -} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc index 361d27298b6..b366dee86f0 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc @@ -48,6 +48,7 @@ #include "src/core/ext/filters/client_channel/client_channel_internal.h" #include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" #include "src/core/ext/filters/stateful_session/stateful_session_filter.h" #include "src/core/ext/xds/xds_health_status.h" #include "src/core/lib/address_utils/sockaddr_utils.h" @@ -349,6 +350,9 @@ class XdsOverrideHostLb : public LoadBalancingPolicy { void ShutdownLocked() override; + void ResetState(); + void ReportTransientFailure(absl::Status status); + OrphanablePtr CreateChildPolicyLocked( const ChannelArgs& args); @@ -361,7 +365,7 @@ class XdsOverrideHostLb : public LoadBalancingPolicy { RefCountedPtr subchannel); // Current config from the resolver. - RefCountedPtr config_; + XdsHealthStatusSet override_host_status_set_; // Internal state. bool shutting_down_ = false; @@ -526,6 +530,10 @@ void XdsOverrideHostLb::ShutdownLocked() { gpr_log(GPR_INFO, "[xds_override_host_lb %p] shutting down", this); } shutting_down_ = true; + ResetState(); +} + +void XdsOverrideHostLb::ResetState() { { // Drop subchannel refs after releasing the lock to avoid deadlock. std::vector subchannel_refs_to_drop; @@ -551,6 +559,18 @@ void XdsOverrideHostLb::ShutdownLocked() { picker_.reset(); } +void XdsOverrideHostLb::ReportTransientFailure(absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { + gpr_log(GPR_INFO, + "[xds_override_host_lb %p] reporting TRANSIENT_FAILURE: %s", this, + status.ToString().c_str()); + } + ResetState(); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, status, + MakeRefCounted(status)); +} + void XdsOverrideHostLb::ExitIdleLocked() { if (child_policy_ != nullptr) child_policy_->ExitIdleLocked(); } @@ -590,12 +610,35 @@ absl::Status XdsOverrideHostLb::UpdateLocked(UpdateArgs args) { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { gpr_log(GPR_INFO, "[xds_override_host_lb %p] Received update", this); } - auto old_config = std::move(config_); - // Update config. - config_ = args.config.TakeAsSubclass(); - if (config_ == nullptr) { + // Grab new LB policy config. + if (args.config == nullptr) { return absl::InvalidArgumentError("Missing policy config"); } + auto new_config = args.config.TakeAsSubclass(); + // Get xDS config. + auto new_xds_config = + args.args.GetObjectRef(); + if (new_xds_config == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError( + "xDS config not passed to xds_cluster_impl LB policy"); + ReportTransientFailure(status); + return status; + } + auto it = new_xds_config->clusters.find(new_config->cluster_name()); + if (it == new_xds_config->clusters.end() || !it->second.ok() || + it->second->cluster == nullptr) { + // Should never happen. + absl::Status status = absl::InternalError(absl::StrCat( + "xDS config has no entry for cluster ", new_config->cluster_name())); + ReportTransientFailure(status); + return status; + } + override_host_status_set_ = it->second->cluster->override_host_statuses; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { + gpr_log(GPR_INFO, "[xds_override_host_lb %p] override host status set: %s", + this, override_host_status_set_.ToString().c_str()); + } // Update address map and wrap endpoint iterator for child policy. if (args.addresses.ok()) { UpdateAddressMap(**args.addresses); @@ -615,7 +658,7 @@ absl::Status XdsOverrideHostLb::UpdateLocked(UpdateArgs args) { UpdateArgs update_args; update_args.addresses = std::move(args.addresses); update_args.resolution_note = std::move(args.resolution_note); - update_args.config = config_->child_config(); + update_args.config = new_config->child_config(); update_args.args = std::move(args.args); if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { gpr_log(GPR_INFO, @@ -627,9 +670,8 @@ absl::Status XdsOverrideHostLb::UpdateLocked(UpdateArgs args) { void XdsOverrideHostLb::MaybeUpdatePickerLocked() { if (picker_ != nullptr) { - auto xds_override_host_picker = - MakeRefCounted(RefAsSubclass(), picker_, - config_->override_host_status_set()); + auto xds_override_host_picker = MakeRefCounted( + RefAsSubclass(), picker_, override_host_status_set_); if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { gpr_log(GPR_INFO, "[xds_override_host_lb %p] updating connectivity: state=%s " @@ -679,7 +721,7 @@ void XdsOverrideHostLb::UpdateAddressMap( XdsHealthStatus status = GetEndpointHealthStatus(endpoint); // Skip draining hosts if not in the override status set. if (status.status() == XdsHealthStatus::kDraining && - !config_->override_host_status_set().Contains(status)) { + !override_host_status_set_.Contains(status)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { gpr_log(GPR_INFO, "[xds_override_host_lb %p] endpoint %s: draining but not in " @@ -795,6 +837,14 @@ XdsOverrideHostLb::AdoptSubchannel( RefCountedPtr XdsOverrideHostLb::Helper::CreateSubchannel( const grpc_resolved_address& address, const ChannelArgs& per_address_args, const ChannelArgs& args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_override_host_trace)) { + auto key = grpc_sockaddr_to_string(&address, /*normalize=*/false); + gpr_log(GPR_INFO, + "[xds_override_host_lb %p] creating subchannel for %s, " + "per_address_args=%s, args=%s", + this, key.value_or("").c_str(), + per_address_args.ToString().c_str(), args.ToString().c_str()); + } auto subchannel = parent()->channel_control_helper()->CreateSubchannel( address, per_address_args, args); return parent()->AdoptSubchannel(address, std::move(subchannel)); @@ -921,50 +971,25 @@ const JsonLoaderInterface* XdsOverrideHostLbConfig::JsonLoader( static const auto kJsonLoader = JsonObjectLoader() // Child policy config is parsed in JsonPostLoad + .Field("clusterName", &XdsOverrideHostLbConfig::cluster_name_) .Finish(); return kJsonLoader; } -void XdsOverrideHostLbConfig::JsonPostLoad(const Json& json, - const JsonArgs& args, +void XdsOverrideHostLbConfig::JsonPostLoad(const Json& json, const JsonArgs&, ValidationErrors* errors) { - { - ValidationErrors::ScopedField field(errors, ".childPolicy"); - auto it = json.object().find("childPolicy"); - if (it == json.object().end()) { - errors->AddError("field not present"); - } else { - auto child_policy_config = CoreConfiguration::Get() - .lb_policy_registry() - .ParseLoadBalancingConfig(it->second); - if (!child_policy_config.ok()) { - errors->AddError(child_policy_config.status().message()); - } else { - child_config_ = std::move(*child_policy_config); - } - } - } - { - ValidationErrors::ScopedField field(errors, ".overrideHostStatus"); - auto host_status_list = LoadJsonObjectField>( - json.object(), args, "overrideHostStatus", errors, - /*required=*/false); - if (host_status_list.has_value()) { - for (size_t i = 0; i < host_status_list->size(); ++i) { - const std::string& host_status = (*host_status_list)[i]; - auto status = XdsHealthStatus::FromString(host_status); - if (!status.has_value()) { - ValidationErrors::ScopedField field(errors, - absl::StrCat("[", i, "]")); - errors->AddError("invalid host status"); - } else { - override_host_status_set_.Add(*status); - } - } + ValidationErrors::ScopedField field(errors, ".childPolicy"); + auto it = json.object().find("childPolicy"); + if (it == json.object().end()) { + errors->AddError("field not present"); + } else { + auto child_policy_config = + CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( + it->second); + if (!child_policy_config.ok()) { + errors->AddError(child_policy_config.status().message()); } else { - override_host_status_set_ = XdsHealthStatusSet( - {XdsHealthStatus(XdsHealthStatus::HealthStatus::kHealthy), - XdsHealthStatus(XdsHealthStatus::HealthStatus::kUnknown)}); + child_config_ = std::move(*child_policy_config); } } } diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h b/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h index 5e0d6dbb773..2cf33345421 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h @@ -21,7 +21,6 @@ #include "absl/strings/string_view.h" -#include "src/core/ext/xds/xds_health_status.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/json/json.h" @@ -46,21 +45,18 @@ class XdsOverrideHostLbConfig : public LoadBalancingPolicy::Config { absl::string_view name() const override { return Name(); } + const std::string& cluster_name() const { return cluster_name_; } RefCountedPtr child_config() const { return child_config_; } - XdsHealthStatusSet override_host_status_set() const { - return override_host_status_set_; - } - static const JsonLoaderInterface* JsonLoader(const JsonArgs&); void JsonPostLoad(const Json& json, const JsonArgs&, ValidationErrors* errors); private: + std::string cluster_name_; RefCountedPtr child_config_; - XdsHealthStatusSet override_host_status_set_; }; } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc b/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc new file mode 100644 index 00000000000..da0ccdafaea --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc @@ -0,0 +1,1039 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" + +#include "absl/strings/str_join.h" + +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h" +#include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h" +#include "src/core/ext/xds/xds_routing.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/match.h" + +namespace grpc_core { + +namespace { + +// Max depth of aggregate cluster dependency graph. +constexpr int kMaxXdsAggregateClusterRecursionDepth = 16; + +} // namespace + +// +// XdsDependencyManager::XdsConfig::ClusterConfig +// + +XdsDependencyManager::XdsConfig::ClusterConfig::ClusterConfig( + std::string cluster_name, std::shared_ptr cluster, + std::shared_ptr endpoints, + std::string resolution_note) + : cluster_name(std::move(cluster_name)), + cluster(std::move(cluster)), + children(absl::in_place_type_t(), std::move(endpoints), + std::move(resolution_note)) {} + +XdsDependencyManager::XdsConfig::ClusterConfig::ClusterConfig( + std::string cluster_name, std::shared_ptr cluster, + std::vector leaf_clusters) + : cluster_name(std::move(cluster_name)), + cluster(std::move(cluster)), + children(absl::in_place_type_t(), + std::move(leaf_clusters)) {} + +// +// XdsDependencyManager::XdsConfig +// + +std::string XdsDependencyManager::XdsConfig::ToString() const { + std::vector parts = { + "{\n listener: {", listener->ToString(), + "}\n route_config: {", route_config->ToString(), + "}\n virtual_host: {", virtual_host->ToString(), + "}\n clusters: {\n"}; + for (const auto& p : clusters) { + parts.push_back(absl::StrCat(" \"", p.first, "\": ")); + if (!p.second.ok()) { + parts.push_back(p.second.status().ToString()); + parts.push_back("\n"); + } else { + parts.push_back( + absl::StrCat(" {\n" + " name: \"", + p.second->cluster_name, + "\"\n" + " cluster: {", + p.second->cluster->ToString(), "}\n")); + Match( + p.second->children, + [&](const ClusterConfig::EndpointConfig& endpoint_config) { + parts.push_back( + absl::StrCat(" endpoints: {", + endpoint_config.endpoints == nullptr + ? "" + : endpoint_config.endpoints->ToString(), + "}\n" + " resolution_note: \"", + endpoint_config.resolution_note, "\"\n")); + }, + [&](const ClusterConfig::AggregateConfig& aggregate_config) { + parts.push_back(absl::StrCat( + " leaf_clusters: [", + absl::StrJoin(aggregate_config.leaf_clusters, ", "), "]\n")); + }); + parts.push_back( + " }\n" + " ]\n"); + } + } + parts.push_back(" }\n}"); + return absl::StrJoin(parts, ""); +} + +// +// XdsDependencyManager::ListenerWatcher +// + +class XdsDependencyManager::ListenerWatcher + : public XdsListenerResourceType::WatcherInterface { + public: + explicit ListenerWatcher(RefCountedPtr dependency_mgr) + : dependency_mgr_(std::move(dependency_mgr)) {} + + void OnResourceChanged( + std::shared_ptr listener, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [dependency_mgr = dependency_mgr_, listener = std::move(listener), + read_delay_handle = std::move(read_delay_handle)]() mutable { + dependency_mgr->OnListenerUpdate(std::move(listener)); + }, + DEBUG_LOCATION); + } + + void OnError( + absl::Status status, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [dependency_mgr = dependency_mgr_, status = std::move(status), + read_delay_handle = std::move(read_delay_handle)]() mutable { + dependency_mgr->OnError(dependency_mgr->listener_resource_name_, + std::move(status)); + }, + DEBUG_LOCATION); + } + + void OnResourceDoesNotExist( + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [dependency_mgr = dependency_mgr_, + read_delay_handle = std::move(read_delay_handle)]() { + dependency_mgr->OnResourceDoesNotExist( + absl::StrCat(dependency_mgr->listener_resource_name_, + ": xDS listener resource does not exist")); + }, + DEBUG_LOCATION); + } + + private: + RefCountedPtr dependency_mgr_; +}; + +// +// XdsDependencyManager::RouteConfigWatcher +// + +class XdsDependencyManager::RouteConfigWatcher + : public XdsRouteConfigResourceType::WatcherInterface { + public: + RouteConfigWatcher(RefCountedPtr dependency_mgr, + std::string name) + : dependency_mgr_(std::move(dependency_mgr)), name_(std::move(name)) {} + + void OnResourceChanged( + std::shared_ptr route_config, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), + route_config = std::move(route_config), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnRouteConfigUpdate(self->name_, + std::move(route_config)); + }, + DEBUG_LOCATION); + } + + void OnError( + absl::Status status, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), status = std::move(status), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnError(self->name_, std::move(status)); + }, + DEBUG_LOCATION); + } + + void OnResourceDoesNotExist( + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), + read_delay_handle = std::move(read_delay_handle)]() { + self->dependency_mgr_->OnResourceDoesNotExist(absl::StrCat( + self->name_, + ": xDS route configuration resource does not exist")); + }, + DEBUG_LOCATION); + } + + private: + RefCountedPtr dependency_mgr_; + std::string name_; +}; + +// +// XdsDependencyManager::ClusterWatcher +// + +class XdsDependencyManager::ClusterWatcher + : public XdsClusterResourceType::WatcherInterface { + public: + ClusterWatcher(RefCountedPtr dependency_mgr, + absl::string_view name) + : dependency_mgr_(std::move(dependency_mgr)), name_(name) {} + + void OnResourceChanged( + std::shared_ptr cluster, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), cluster = std::move(cluster), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnClusterUpdate(self->name_, + std::move(cluster)); + }, + DEBUG_LOCATION); + } + + void OnError( + absl::Status status, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), status = std::move(status), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnClusterError(self->name_, std::move(status)); + }, + DEBUG_LOCATION); + } + + void OnResourceDoesNotExist( + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), + read_delay_handle = std::move(read_delay_handle)]() { + self->dependency_mgr_->OnClusterDoesNotExist(self->name_); + }, + DEBUG_LOCATION); + } + + private: + RefCountedPtr dependency_mgr_; + std::string name_; +}; + +// +// XdsDependencyManager::EndpointWatcher +// + +class XdsDependencyManager::EndpointWatcher + : public XdsEndpointResourceType::WatcherInterface { + public: + EndpointWatcher(RefCountedPtr dependency_mgr, + absl::string_view name) + : dependency_mgr_(std::move(dependency_mgr)), name_(name) {} + + void OnResourceChanged( + std::shared_ptr endpoint, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), + endpoint = std::move(endpoint), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnEndpointUpdate(self->name_, + std::move(endpoint)); + }, + DEBUG_LOCATION); + } + + void OnError( + absl::Status status, + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), status = std::move(status), + read_delay_handle = std::move(read_delay_handle)]() mutable { + self->dependency_mgr_->OnEndpointError(self->name_, + std::move(status)); + }, + DEBUG_LOCATION); + } + + void OnResourceDoesNotExist( + RefCountedPtr read_delay_handle) override { + dependency_mgr_->work_serializer_->Run( + [self = RefAsSubclass(), + read_delay_handle = std::move(read_delay_handle)]() { + self->dependency_mgr_->OnEndpointDoesNotExist(self->name_); + }, + DEBUG_LOCATION); + } + + private: + RefCountedPtr dependency_mgr_; + std::string name_; +}; + +// +// XdsDependencyManager::DnsResultHandler +// + +class XdsDependencyManager::DnsResultHandler : public Resolver::ResultHandler { + public: + DnsResultHandler(RefCountedPtr dependency_mgr, + std::string name) + : dependency_mgr_(std::move(dependency_mgr)), name_(std::move(name)) {} + + void ReportResult(Resolver::Result result) override { + dependency_mgr_->work_serializer_->Run( + [dependency_mgr = dependency_mgr_, name = name_, + result = std::move(result)]() mutable { + dependency_mgr->OnDnsResult(name, std::move(result)); + }, + DEBUG_LOCATION); + } + + private: + RefCountedPtr dependency_mgr_; + std::string name_; +}; + +// +// XdsDependencyManager::ClusterSubscription +// + +void XdsDependencyManager::ClusterSubscription::Orphan() { + dependency_mgr_->work_serializer_->Run( + [self = WeakRef()]() { + self->dependency_mgr_->OnClusterSubscriptionUnref(self->cluster_name_, + self.get()); + }, + DEBUG_LOCATION); +} + +// +// XdsDependencyManager +// + +XdsDependencyManager::XdsDependencyManager( + RefCountedPtr xds_client, + std::shared_ptr work_serializer, + std::unique_ptr watcher, std::string data_plane_authority, + std::string listener_resource_name, ChannelArgs args, + grpc_pollset_set* interested_parties) + : xds_client_(std::move(xds_client)), + work_serializer_(std::move(work_serializer)), + watcher_(std::move(watcher)), + data_plane_authority_(std::move(data_plane_authority)), + listener_resource_name_(std::move(listener_resource_name)), + args_(std::move(args)), + interested_parties_(interested_parties) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] starting watch for listener %s", this, + listener_resource_name_.c_str()); + } + auto listener_watcher = MakeRefCounted(Ref()); + listener_watcher_ = listener_watcher.get(); + XdsListenerResourceType::StartWatch( + xds_client_.get(), listener_resource_name_, std::move(listener_watcher)); +} + +void XdsDependencyManager::Orphan() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] shutting down", this); + } + if (listener_watcher_ != nullptr) { + XdsListenerResourceType::CancelWatch( + xds_client_.get(), listener_resource_name_, listener_watcher_, + /*delay_unsubscription=*/false); + } + if (route_config_watcher_ != nullptr) { + XdsRouteConfigResourceType::CancelWatch( + xds_client_.get(), route_config_name_, route_config_watcher_, + /*delay_unsubscription=*/false); + } + for (const auto& p : cluster_watchers_) { + XdsClusterResourceType::CancelWatch(xds_client_.get(), p.first, + p.second.watcher, + /*delay_unsubscription=*/false); + } + for (const auto& p : endpoint_watchers_) { + XdsEndpointResourceType::CancelWatch(xds_client_.get(), p.first, + p.second.watcher, + /*delay_unsubscription=*/false); + } + cluster_subscriptions_.clear(); + xds_client_.reset(); + for (auto& p : dns_resolvers_) { + p.second.resolver.reset(); + } + Unref(); +} + +void XdsDependencyManager::OnListenerUpdate( + std::shared_ptr listener) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] received Listener update", + this); + } + if (xds_client_ == nullptr) return; + const auto* hcm = absl::get_if( + &listener->listener); + if (hcm == nullptr) { + return OnError(listener_resource_name_, + absl::UnavailableError("not an API listener")); + } + current_listener_ = std::move(listener); + Match( + hcm->route_config, + // RDS resource name + [&](const std::string& rds_name) { + // If the RDS name changed, update the RDS watcher. + // Note that this will be true on the initial update, because + // route_config_name_ will be empty. + if (route_config_name_ != rds_name) { + // If we already had a watch (i.e., if the previous config had + // a different RDS name), stop the previous watch. + // There will be no previous watch if either (a) this is the + // initial resource update or (b) the previous Listener had an + // inlined RouteConfig. + if (route_config_watcher_ != nullptr) { + XdsRouteConfigResourceType::CancelWatch( + xds_client_.get(), route_config_name_, route_config_watcher_, + /*delay_unsubscription=*/true); + route_config_watcher_ = nullptr; + } + // Start watch for the new RDS resource name. + route_config_name_ = rds_name; + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log( + GPR_INFO, + "[XdsDependencyManager %p] starting watch for route config %s", + this, route_config_name_.c_str()); + } + auto watcher = + MakeRefCounted(Ref(), route_config_name_); + route_config_watcher_ = watcher.get(); + XdsRouteConfigResourceType::StartWatch( + xds_client_.get(), route_config_name_, std::move(watcher)); + } else { + // RDS resource name has not changed, so no watch needs to be + // updated, but we still need to propagate any changes in the + // HCM config (e.g., the list of HTTP filters). + MaybeReportUpdate(); + } + }, + // inlined RouteConfig + [&](const std::shared_ptr& route_config) { + // If the previous update specified an RDS resource instead of + // having an inlined RouteConfig, we need to cancel the RDS watch. + if (route_config_watcher_ != nullptr) { + XdsRouteConfigResourceType::CancelWatch( + xds_client_.get(), route_config_name_, route_config_watcher_); + route_config_watcher_ = nullptr; + route_config_name_.clear(); + } + OnRouteConfigUpdate("", route_config); + }); +} + +namespace { + +class XdsVirtualHostListIterator : public XdsRouting::VirtualHostListIterator { + public: + explicit XdsVirtualHostListIterator( + const std::vector* virtual_hosts) + : virtual_hosts_(virtual_hosts) {} + + size_t Size() const override { return virtual_hosts_->size(); } + + const std::vector& GetDomainsForVirtualHost( + size_t index) const override { + return (*virtual_hosts_)[index].domains; + } + + private: + const std::vector* virtual_hosts_; +}; + +// Gets the set of clusters referenced in the specified virtual host. +absl::flat_hash_set GetClustersFromVirtualHost( + const XdsRouteConfigResource::VirtualHost& virtual_host) { + absl::flat_hash_set clusters; + for (auto& route : virtual_host.routes) { + auto* route_action = + absl::get_if(&route.action); + if (route_action == nullptr) continue; + Match( + route_action->action, + // cluster name + [&](const XdsRouteConfigResource::Route::RouteAction::ClusterName& + cluster_name) { clusters.insert(cluster_name.cluster_name); }, + // WeightedClusters + [&](const std::vector< + XdsRouteConfigResource::Route::RouteAction::ClusterWeight>& + weighted_clusters) { + for (const auto& weighted_cluster : weighted_clusters) { + clusters.insert(weighted_cluster.name); + } + }, + // ClusterSpecifierPlugin + [&](const XdsRouteConfigResource::Route::RouteAction:: + ClusterSpecifierPluginName&) { + // Clusters are determined dynamically in this case, so we + // can't add any clusters here. + }); + } + return clusters; +} + +} // namespace + +void XdsDependencyManager::OnRouteConfigUpdate( + const std::string& name, + std::shared_ptr route_config) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] received RouteConfig update for %s", + this, name.empty() ? "" : name.c_str()); + } + if (xds_client_ == nullptr) return; + // Ignore updates for stale names. + if (name.empty()) { + if (!route_config_name_.empty()) return; + } else { + if (name != route_config_name_) return; + } + // Find the relevant VirtualHost from the RouteConfiguration. + // If the resource doesn't have the right vhost, fail without updating + // our data. + auto vhost_index = XdsRouting::FindVirtualHostForDomain( + XdsVirtualHostListIterator(&route_config->virtual_hosts), + data_plane_authority_); + if (!vhost_index.has_value()) { + OnError(route_config_name_.empty() ? listener_resource_name_ + : route_config_name_, + absl::UnavailableError( + absl::StrCat("could not find VirtualHost for ", + data_plane_authority_, " in RouteConfiguration"))); + return; + } + // Update our data. + current_route_config_ = std::move(route_config); + current_virtual_host_ = ¤t_route_config_->virtual_hosts[*vhost_index]; + clusters_from_route_config_ = + GetClustersFromVirtualHost(*current_virtual_host_); + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnError(std::string context, absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] received Listener or RouteConfig " + "error: %s %s", + this, context.c_str(), status.ToString().c_str()); + } + if (xds_client_ == nullptr) return; + if (current_virtual_host_ != nullptr) return; + watcher_->OnError(context, std::move(status)); +} + +void XdsDependencyManager::OnResourceDoesNotExist(std::string context) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] %s", this, context.c_str()); + } + if (xds_client_ == nullptr) return; + current_virtual_host_ = nullptr; + watcher_->OnResourceDoesNotExist(std::move(context)); +} + +void XdsDependencyManager::OnClusterUpdate( + const std::string& name, + std::shared_ptr cluster) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] received Cluster update: %s", + this, name.c_str()); + } + if (xds_client_ == nullptr) return; + auto it = cluster_watchers_.find(name); + if (it == cluster_watchers_.end()) return; + it->second.update = std::move(cluster); + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnClusterError(const std::string& name, + absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] received Cluster error: %s %s", + this, name.c_str(), status.ToString().c_str()); + } + if (xds_client_ == nullptr) return; + auto it = cluster_watchers_.find(name); + if (it == cluster_watchers_.end()) return; + if (it->second.update.value_or(nullptr) == nullptr) { + it->second.update = + absl::Status(status.code(), absl::StrCat(name, ": ", status.message())); + } + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnClusterDoesNotExist(const std::string& name) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] Cluster does not exist: %s", + this, name.c_str()); + } + if (xds_client_ == nullptr) return; + auto it = cluster_watchers_.find(name); + if (it == cluster_watchers_.end()) return; + it->second.update = absl::UnavailableError( + absl::StrCat("CDS resource ", name, " does not exist")); + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnEndpointUpdate( + const std::string& name, + std::shared_ptr endpoint) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] received Endpoint update: %s", + this, name.c_str()); + } + if (xds_client_ == nullptr) return; + auto it = endpoint_watchers_.find(name); + if (it == endpoint_watchers_.end()) return; + if (endpoint->priorities.empty()) { + it->second.update.resolution_note = + absl::StrCat("EDS resource ", name, " contains no localities"); + } else { + std::set empty_localities; + for (const auto& priority : endpoint->priorities) { + for (const auto& p : priority.localities) { + if (p.second.endpoints.empty()) { + empty_localities.insert(p.first->AsHumanReadableString()); + } + } + } + if (!empty_localities.empty()) { + it->second.update.resolution_note = + absl::StrCat("EDS resource ", name, " contains empty localities: [", + absl::StrJoin(empty_localities, "; "), "]"); + } + } + it->second.update.endpoints = std::move(endpoint); + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnEndpointError(const std::string& name, + absl::Status status) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] received Endpoint error: %s %s", this, + name.c_str(), status.ToString().c_str()); + } + if (xds_client_ == nullptr) return; + auto it = endpoint_watchers_.find(name); + if (it == endpoint_watchers_.end()) return; + if (it->second.update.endpoints == nullptr) { + it->second.update.resolution_note = + absl::StrCat("EDS resource ", name, ": ", status.ToString()); + MaybeReportUpdate(); + } +} + +void XdsDependencyManager::OnEndpointDoesNotExist(const std::string& name) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] Endpoint does not exist: %s", + this, name.c_str()); + } + if (xds_client_ == nullptr) return; + auto it = endpoint_watchers_.find(name); + if (it == endpoint_watchers_.end()) return; + it->second.update.endpoints.reset(); + it->second.update.resolution_note = + absl::StrCat("EDS resource ", name, " does not exist"); + MaybeReportUpdate(); +} + +void XdsDependencyManager::OnDnsResult(const std::string& dns_name, + Resolver::Result result) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] received DNS update: %s", this, + dns_name.c_str()); + } + if (xds_client_ == nullptr) return; + auto it = dns_resolvers_.find(dns_name); + if (it == dns_resolvers_.end()) return; + PopulateDnsUpdate(dns_name, std::move(result), &it->second); + MaybeReportUpdate(); +} + +void XdsDependencyManager::PopulateDnsUpdate(const std::string& dns_name, + Resolver::Result result, + DnsState* dns_state) { + // Convert resolver result to EDS update. + XdsEndpointResource::Priority::Locality locality; + locality.name = MakeRefCounted("", "", ""); + locality.lb_weight = 1; + if (result.addresses.ok()) { + locality.endpoints = std::move(*result.addresses); + dns_state->update.resolution_note = std::move(result.resolution_note); + } else if (result.resolution_note.empty()) { + dns_state->update.resolution_note = + absl::StrCat("DNS resolution failed for ", dns_name, ": ", + result.addresses.status().ToString()); + } + XdsEndpointResource::Priority priority; + priority.localities.emplace(locality.name.get(), std::move(locality)); + auto resource = std::make_shared(); + resource->priorities.emplace_back(std::move(priority)); + dns_state->update.endpoints = std::move(resource); +} + +bool XdsDependencyManager::PopulateClusterConfigMap( + absl::string_view name, int depth, + absl::flat_hash_map>* + cluster_config_map, + std::set* eds_resources_seen, + std::set* dns_names_seen, + absl::StatusOr>* leaf_clusters) { + if (depth > 0) GPR_ASSERT(leaf_clusters != nullptr); + if (depth == kMaxXdsAggregateClusterRecursionDepth) { + *leaf_clusters = + absl::UnavailableError("aggregate cluster graph exceeds max depth"); + return true; + } + // Don't process the cluster again if we've already seen it in some + // other branch of the recursion tree. We populate it with a non-OK + // status here, since we need an entry in the map to avoid incorrectly + // stopping the CDS watch, but we'll overwrite this below if we actually + // have the data for the cluster. + auto p = cluster_config_map->emplace( + name, absl::InternalError("cluster data not yet available")); + if (!p.second) return true; + auto& cluster_config = p.first->second; + auto& state = cluster_watchers_[name]; + // Create a new watcher if needed. + if (state.watcher == nullptr) { + auto watcher = MakeRefCounted(Ref(), name); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] starting watch for cluster %s", this, + std::string(name).c_str()); + } + state.watcher = watcher.get(); + XdsClusterResourceType::StartWatch(xds_client_.get(), name, + std::move(watcher)); + return false; + } + // If there was an error fetching the CDS resource, report the error. + if (!state.update.ok()) { + cluster_config = state.update.status(); + return true; + } + // If we don't have the resource yet, we can't return a config yet. + if (*state.update == nullptr) return false; + // Populate endpoint info based on cluster type. + return Match( + (*state.update)->type, + // EDS cluster. + [&](const XdsClusterResource::Eds& eds) { + absl::string_view eds_resource_name = + eds.eds_service_name.empty() ? name : eds.eds_service_name; + eds_resources_seen->insert(eds_resource_name); + // Start EDS watch if needed. + auto& eds_state = endpoint_watchers_[eds_resource_name]; + if (eds_state.watcher == nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] starting watch for endpoint %s", + this, std::string(eds_resource_name).c_str()); + } + auto watcher = + MakeRefCounted(Ref(), eds_resource_name); + eds_state.watcher = watcher.get(); + XdsEndpointResourceType::StartWatch( + xds_client_.get(), eds_resource_name, std::move(watcher)); + return false; + } + // Check if EDS resource has been returned. + if (eds_state.update.endpoints == nullptr && + eds_state.update.resolution_note.empty()) { + return false; + } + // Populate cluster config. + cluster_config.emplace(std::string(name), *state.update, + eds_state.update.endpoints, + eds_state.update.resolution_note); + if (leaf_clusters != nullptr) (*leaf_clusters)->push_back(name); + return true; + }, + // LOGICAL_DNS cluster. + [&](const XdsClusterResource::LogicalDns& logical_dns) { + dns_names_seen->insert(logical_dns.hostname); + // Start DNS resolver if needed. + auto& dns_state = dns_resolvers_[logical_dns.hostname]; + if (dns_state.resolver == nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] starting DNS resolver for %s", + this, logical_dns.hostname.c_str()); + } + auto* fake_resolver_response_generator = args_.GetPointer< + FakeResolverResponseGenerator>( + GRPC_ARG_XDS_LOGICAL_DNS_CLUSTER_FAKE_RESOLVER_RESPONSE_GENERATOR); + ChannelArgs args = args_; + std::string target; + if (fake_resolver_response_generator != nullptr) { + target = absl::StrCat("fake:", logical_dns.hostname); + args = args.SetObject(fake_resolver_response_generator->Ref()); + } else { + target = absl::StrCat("dns:", logical_dns.hostname); + } + dns_state.resolver = + CoreConfiguration::Get().resolver_registry().CreateResolver( + target, args, interested_parties_, work_serializer_, + std::make_unique(Ref(), + logical_dns.hostname)); + if (dns_state.resolver == nullptr) { + Resolver::Result result; + result.addresses.emplace(); // Empty list. + result.resolution_note = absl::StrCat( + "failed to create DNS resolver for ", logical_dns.hostname); + PopulateDnsUpdate(logical_dns.hostname, std::move(result), + &dns_state); + } else { + dns_state.resolver->StartLocked(); + return false; + } + } + // Check if result has been returned. + if (dns_state.update.endpoints == nullptr && + dns_state.update.resolution_note.empty()) { + return false; + } + // Populate cluster config. + cluster_config.emplace(std::string(name), *state.update, + dns_state.update.endpoints, + dns_state.update.resolution_note); + if (leaf_clusters != nullptr) (*leaf_clusters)->push_back(name); + return true; + }, + // Aggregate cluster. Recursively expand to child clusters. + [&](const XdsClusterResource::Aggregate& aggregate) { + // Grab a ref to the CDS resource for the aggregate cluster here, + // since our reference into cluster_watchers_ will be invalidated + // when we recursively call ourselves and add entries to the + // map for underlying clusters. + auto cluster_resource = *state.update; + // Recursively expand leaf clusters. + absl::StatusOr> child_leaf_clusters; + child_leaf_clusters.emplace(); + bool have_all_resources = true; + for (const std::string& child_name : + aggregate.prioritized_cluster_names) { + have_all_resources &= PopulateClusterConfigMap( + child_name, depth + 1, cluster_config_map, eds_resources_seen, + dns_names_seen, &child_leaf_clusters); + if (!child_leaf_clusters.ok()) break; + } + // Note that we cannot use the cluster_config reference we + // created above, because it may have been invalidated by map + // insertions when we recursively called ourselves, so we have + // to do the lookup in cluster_config_map again. + auto& aggregate_cluster_config = (*cluster_config_map)[name]; + // If we exceeded max recursion depth, report an error for the + // cluster, and propagate the error up if needed. + if (!child_leaf_clusters.ok()) { + aggregate_cluster_config = child_leaf_clusters.status(); + if (leaf_clusters != nullptr) { + *leaf_clusters = child_leaf_clusters.status(); + } + return true; + } + // If needed, propagate leaf cluster list up the tree. + if (leaf_clusters != nullptr) { + (*leaf_clusters) + ->insert((*leaf_clusters)->end(), child_leaf_clusters->begin(), + child_leaf_clusters->end()); + } + // If there are no leaf clusters, report an error for the cluster. + if (have_all_resources && child_leaf_clusters->empty()) { + aggregate_cluster_config = absl::UnavailableError( + absl::StrCat("aggregate cluster dependency graph for ", name, + " has no leaf clusters")); + return true; + } + // Populate cluster config. + // Note that we do this even for aggregate clusters that are not + // at the root of the tree, because we need to make sure the list + // of underlying cluster names stays alive so that the leaf cluster + // list of the root aggregate cluster can point to those strings. + aggregate_cluster_config.emplace(std::string(name), + std::move(cluster_resource), + std::move(*child_leaf_clusters)); + return have_all_resources; + }); +} + +RefCountedPtr +XdsDependencyManager::GetClusterSubscription(absl::string_view cluster_name) { + auto it = cluster_subscriptions_.find(cluster_name); + if (it != cluster_subscriptions_.end()) { + auto subscription = it->second->RefIfNonZero(); + if (subscription != nullptr) return subscription; + } + auto subscription = MakeRefCounted(cluster_name, Ref()); + cluster_subscriptions_.emplace(subscription->cluster_name(), + subscription->WeakRef()); + // If the cluster is not already subscribed to by virtue of being + // referenced in the route config, then trigger the CDS watch. + if (!clusters_from_route_config_.contains(cluster_name)) { + MaybeReportUpdate(); + } + return subscription; +} + +void XdsDependencyManager::OnClusterSubscriptionUnref( + absl::string_view cluster_name, ClusterSubscription* subscription) { + auto it = cluster_subscriptions_.find(cluster_name); + // Shouldn't happen, but ignore if it does. + if (it == cluster_subscriptions_.end()) return; + // Do nothing if the subscription has already been replaced. + if (it->second != subscription) return; + // Remove the entry. + cluster_subscriptions_.erase(it); + // If this cluster is not already subscribed to by virtue of being + // referenced in the route config, then update watches and generate a + // new update. + if (!clusters_from_route_config_.contains(cluster_name)) { + MaybeReportUpdate(); + } +} + +void XdsDependencyManager::MaybeReportUpdate() { + // Populate Listener and RouteConfig fields. + if (current_virtual_host_ == nullptr) return; + auto config = MakeRefCounted(); + config->listener = current_listener_; + config->route_config = current_route_config_; + config->virtual_host = current_virtual_host_; + // Determine the set of clusters we should be watching. + std::set clusters_to_watch; + for (const absl::string_view& cluster : clusters_from_route_config_) { + clusters_to_watch.insert(cluster); + } + for (const auto& p : cluster_subscriptions_) { + clusters_to_watch.insert(p.first); + } + // Populate Cluster map. + // We traverse the entire graph even if we don't yet have all of the + // resources we need to ensure that the right set of watches are active. + std::set eds_resources_seen; + std::set dns_names_seen; + bool have_all_resources = true; + for (const absl::string_view& cluster : clusters_to_watch) { + have_all_resources &= PopulateClusterConfigMap( + cluster, 0, &config->clusters, &eds_resources_seen, &dns_names_seen); + } + // Remove entries in cluster_watchers_ for any clusters not in + // config->clusters. + for (auto it = cluster_watchers_.begin(); it != cluster_watchers_.end();) { + const std::string& cluster_name = it->first; + if (config->clusters.contains(cluster_name)) { + ++it; + continue; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] cancelling watch for cluster %s", this, + cluster_name.c_str()); + } + XdsClusterResourceType::CancelWatch(xds_client_.get(), cluster_name, + it->second.watcher, + /*delay_unsubscription=*/false); + cluster_watchers_.erase(it++); + } + // Remove entries in endpoint_watchers_ for any EDS resources not in + // eds_resources_seen. + for (auto it = endpoint_watchers_.begin(); it != endpoint_watchers_.end();) { + const std::string& eds_resource_name = it->first; + if (eds_resources_seen.find(eds_resource_name) != + eds_resources_seen.end()) { + ++it; + continue; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] cancelling watch for EDS resource %s", + this, eds_resource_name.c_str()); + } + XdsEndpointResourceType::CancelWatch(xds_client_.get(), eds_resource_name, + it->second.watcher, + /*delay_unsubscription=*/false); + endpoint_watchers_.erase(it++); + } + // Remove entries in dns_resolvers_ for any DNS name not in + // eds_resources_seen. + for (auto it = dns_resolvers_.begin(); it != dns_resolvers_.end();) { + const std::string& dns_name = it->first; + if (dns_names_seen.find(dns_name) != dns_names_seen.end()) { + ++it; + continue; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] shutting down DNS resolver for %s", + this, dns_name.c_str()); + } + dns_resolvers_.erase(it++); + } + // If we have all the data we need, then send an update. + if (!have_all_resources) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, + "[XdsDependencyManager %p] missing data -- NOT returning config", + this); + } + return; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { + gpr_log(GPR_INFO, "[XdsDependencyManager %p] returning config: %s", this, + config->ToString().c_str()); + } + watcher_->OnUpdate(std::move(config)); +} + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h b/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h new file mode 100644 index 00000000000..697098ae028 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h @@ -0,0 +1,281 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_DEPENDENCY_MANAGER_H +#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_DEPENDENCY_MANAGER_H + +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/string_view.h" + +#include "src/core/ext/xds/xds_client_grpc.h" +#include "src/core/ext/xds/xds_cluster.h" +#include "src/core/ext/xds/xds_endpoint.h" +#include "src/core/ext/xds/xds_listener.h" +#include "src/core/ext/xds/xds_route_config.h" +#include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/resolver/resolver.h" + +namespace grpc_core { + +// Watches all xDS resources and handles dependencies between them. +// Reports updates only when all necessary resources have been obtained. +class XdsDependencyManager : public RefCounted, + public Orphanable { + public: + struct XdsConfig : public RefCounted { + // Listener resource. + std::shared_ptr listener; + // RouteConfig resource. Will be populated even if RouteConfig is + // inlined into the Listener resource. + std::shared_ptr route_config; + // Virtual host. Points into route_config. Will always be non-null. + const XdsRouteConfigResource::VirtualHost* virtual_host; + + // Cluster map. A cluster will have a non-OK status if either + // (a) there was an error and we did not already have a valid + // resource or (b) the resource does not exist. + struct ClusterConfig { + // Cluster name and resource. + std::string cluster_name; + std::shared_ptr cluster; + // Endpoint info. If there was an error, endpoints will be null + // and resolution_note will be set. Not used for aggregate clusters. + struct EndpointConfig { + std::shared_ptr endpoints; + std::string resolution_note; + + EndpointConfig(std::shared_ptr endpoints, + std::string resolution_note) + : endpoints(std::move(endpoints)), + resolution_note(std::move(resolution_note)) {} + bool operator==(const EndpointConfig& other) const { + return endpoints == other.endpoints && + resolution_note == other.resolution_note; + } + }; + // The list of leaf clusters for an aggregate cluster. + struct AggregateConfig { + std::vector leaf_clusters; + + explicit AggregateConfig(std::vector leaf_clusters) + : leaf_clusters(std::move(leaf_clusters)) {} + bool operator==(const AggregateConfig& other) const { + return leaf_clusters == other.leaf_clusters; + } + }; + absl::variant children; + + // Ctor for leaf clusters. + ClusterConfig(std::string cluster_name, + std::shared_ptr cluster, + std::shared_ptr endpoints, + std::string resolution_note); + // Ctor for aggregate clusters. + ClusterConfig(std::string cluster_name, + std::shared_ptr cluster, + std::vector leaf_clusters); + + bool operator==(const ClusterConfig& other) const { + return cluster_name == other.cluster_name && cluster == other.cluster && + children == other.children; + } + }; + absl::flat_hash_map> clusters; + + std::string ToString() const; + + static absl::string_view ChannelArgName() { + return GRPC_ARG_NO_SUBCHANNEL_PREFIX "xds_config"; + } + static int ChannelArgsCompare(const XdsConfig* a, const XdsConfig* b) { + return QsortCompare(a, b); + } + static constexpr bool ChannelArgUseConstPtr() { return true; } + }; + + class Watcher { + public: + virtual ~Watcher() = default; + + virtual void OnUpdate(RefCountedPtr config) = 0; + + // These methods are invoked when there is an error or + // does-not-exist on LDS or RDS only. + virtual void OnError(absl::string_view context, absl::Status status) = 0; + virtual void OnResourceDoesNotExist(std::string context) = 0; + }; + + class ClusterSubscription : public DualRefCounted { + public: + ClusterSubscription(absl::string_view cluster_name, + RefCountedPtr dependency_mgr) + : cluster_name_(cluster_name), + dependency_mgr_(std::move(dependency_mgr)) {} + + void Orphan() override; + + absl::string_view cluster_name() const { return cluster_name_; } + + private: + std::string cluster_name_; + RefCountedPtr dependency_mgr_; + }; + + XdsDependencyManager(RefCountedPtr xds_client, + std::shared_ptr work_serializer, + std::unique_ptr watcher, + std::string data_plane_authority, + std::string listener_resource_name, ChannelArgs args, + grpc_pollset_set* interested_parties); + + void Orphan() override; + + // Gets an external cluster subscription. This allows us to include + // clusters in the config that are referenced by something other than + // the route config (e.g., RLS). The cluster will be included in the + // config as long as the returned object is still referenced. + RefCountedPtr GetClusterSubscription( + absl::string_view cluster_name); + + static absl::string_view ChannelArgName() { + return GRPC_ARG_NO_SUBCHANNEL_PREFIX "xds_dependency_manager"; + } + static int ChannelArgsCompare(const XdsDependencyManager* a, + const XdsDependencyManager* b) { + return QsortCompare(a, b); + } + + private: + class ListenerWatcher; + class RouteConfigWatcher; + class ClusterWatcher; + class EndpointWatcher; + + class DnsResultHandler; + + struct ClusterWatcherState { + // Pointer to watcher, to be used when cancelling. + // Not owned, so do not dereference. + ClusterWatcher* watcher = nullptr; + // Most recent update obtained from this watcher. + absl::StatusOr> update = nullptr; + }; + + struct EndpointConfig { + // If there was an error, update will be null and resolution_note + // will be non-empty. + std::shared_ptr endpoints; + std::string resolution_note; + }; + + struct EndpointWatcherState { + // Pointer to watcher, to be used when cancelling. + // Not owned, so do not dereference. + EndpointWatcher* watcher = nullptr; + // Most recent update obtained from this watcher. + EndpointConfig update; + }; + + struct DnsState { + OrphanablePtr resolver; + // Most recent result from the resolver. + EndpointConfig update; + }; + + // Event handlers. + void OnListenerUpdate(std::shared_ptr listener); + void OnRouteConfigUpdate( + const std::string& name, + std::shared_ptr route_config); + void OnError(std::string context, absl::Status status); + void OnResourceDoesNotExist(std::string context); + + void OnClusterUpdate(const std::string& name, + std::shared_ptr cluster); + void OnClusterError(const std::string& name, absl::Status status); + void OnClusterDoesNotExist(const std::string& name); + + void OnEndpointUpdate(const std::string& name, + std::shared_ptr endpoint); + void OnEndpointError(const std::string& name, absl::Status status); + void OnEndpointDoesNotExist(const std::string& name); + + void OnDnsResult(const std::string& dns_name, Resolver::Result result); + void PopulateDnsUpdate(const std::string& dns_name, Resolver::Result result, + DnsState* dns_state); + + // Starts CDS and EDS/DNS watches for the specified cluster if needed. + // Adds an entry to cluster_config_map, which will contain the cluster + // data if the data is available. + // For each EDS cluster, adds the EDS resource to eds_resources_seen. + // For each Logical DNS cluster, adds the DNS hostname to dns_names_seen. + // For aggregate clusters, calls itself recursively. If leaf_clusters is + // non-null, populates it with a list of leaf clusters, or an error if + // max depth is exceeded. + // Returns true if all resources have been obtained. + bool PopulateClusterConfigMap( + absl::string_view name, int depth, + absl::flat_hash_map>* + cluster_config_map, + std::set* eds_resources_seen, + std::set* dns_names_seen, + absl::StatusOr>* leaf_clusters = nullptr); + + // Called when an external cluster subscription is unreffed. + void OnClusterSubscriptionUnref(absl::string_view cluster_name, + ClusterSubscription* subscription); + + // Checks whether all necessary resources have been obtained, and if + // so reports an update to the watcher. + void MaybeReportUpdate(); + + // Parameters passed into ctor. + RefCountedPtr xds_client_; + std::shared_ptr work_serializer_; + std::unique_ptr watcher_; + const std::string data_plane_authority_; + const std::string listener_resource_name_; + ChannelArgs args_; + grpc_pollset_set* interested_parties_; + + // Listener state. + ListenerWatcher* listener_watcher_ = nullptr; + std::shared_ptr current_listener_; + std::string route_config_name_; + + // RouteConfig state. + RouteConfigWatcher* route_config_watcher_ = nullptr; + std::shared_ptr current_route_config_; + const XdsRouteConfigResource::VirtualHost* current_virtual_host_ = nullptr; + absl::flat_hash_set clusters_from_route_config_; + + // Cluster state. + absl::flat_hash_map cluster_watchers_; + absl::flat_hash_map> + cluster_subscriptions_; + + // Endpoint state. + absl::flat_hash_map endpoint_watchers_; + absl::flat_hash_map dns_resolvers_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_DEPENDENCY_MANAGER_H diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc index 2ae4b1d2d3b..732f07169bc 100644 --- a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc @@ -16,8 +16,6 @@ #include -#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h" - #include #include @@ -53,9 +51,11 @@ #include "src/core/ext/filters/client_channel/client_channel_internal.h" #include "src/core/ext/filters/client_channel/config_selector.h" #include "src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_bootstrap_grpc.h" -#include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_client_grpc.h" #include "src/core/ext/xds/xds_http_filters.h" #include "src/core/ext/xds/xds_listener.h" @@ -96,12 +96,8 @@ namespace grpc_core { -TraceFlag grpc_xds_resolver_trace(false, "xds_resolver"); - namespace { -using ReadDelayHandle = XdsClient::ReadDelayHandle; - // // XdsResolver // @@ -139,87 +135,22 @@ class XdsResolver : public Resolver { } private: - class ListenerWatcher : public XdsListenerResourceType::WatcherInterface { + class XdsWatcher : public XdsDependencyManager::Watcher { public: - explicit ListenerWatcher(RefCountedPtr resolver) + explicit XdsWatcher(RefCountedPtr resolver) : resolver_(std::move(resolver)) {} - void OnResourceChanged( - std::shared_ptr listener, - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), - listener = std::move(listener), - read_delay_handle = std::move(read_delay_handle)]() mutable { - self->resolver_->OnListenerUpdate(std::move(listener)); - }, - DEBUG_LOCATION); - } - void OnError(absl::Status status, - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), status = std::move(status), - read_delay_handle = std::move(read_delay_handle)]() mutable { - self->resolver_->OnError(self->resolver_->lds_resource_name_, - std::move(status)); - }, - DEBUG_LOCATION); - } - void OnResourceDoesNotExist( - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), - read_delay_handle = std::move(read_delay_handle)]() { - self->resolver_->OnResourceDoesNotExist( - absl::StrCat(self->resolver_->lds_resource_name_, - ": xDS listener resource does not exist")); - }, - DEBUG_LOCATION); - } - - private: - RefCountedPtr resolver_; - }; - class RouteConfigWatcher - : public XdsRouteConfigResourceType::WatcherInterface { - public: - explicit RouteConfigWatcher(RefCountedPtr resolver) - : resolver_(std::move(resolver)) {} - void OnResourceChanged( - std::shared_ptr route_config, - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), - route_config = std::move(route_config), - read_delay_handle = std::move(read_delay_handle)]() mutable { - if (self != self->resolver_->route_config_watcher_) return; - self->resolver_->OnRouteConfigUpdate(std::move(route_config)); - }, - DEBUG_LOCATION); + void OnUpdate( + RefCountedPtr config) override { + resolver_->OnUpdate(std::move(config)); } - void OnError(absl::Status status, - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), - status = std::move(status), - read_delay_handle = std::move(read_delay_handle)]() mutable { - if (self != self->resolver_->route_config_watcher_) return; - self->resolver_->OnError(self->resolver_->route_config_name_, - std::move(status)); - }, - DEBUG_LOCATION); + + void OnError(absl::string_view context, absl::Status status) override { + resolver_->OnError(context, std::move(status)); } - void OnResourceDoesNotExist( - RefCountedPtr read_delay_handle) override { - resolver_->work_serializer_->Run( - [self = RefAsSubclass(), - read_delay_handle = std::move(read_delay_handle)]() { - if (self != self->resolver_->route_config_watcher_) return; - self->resolver_->OnResourceDoesNotExist(absl::StrCat( - self->resolver_->route_config_name_, - ": xDS route configuration resource does not exist")); - }, - DEBUG_LOCATION); + + void OnResourceDoesNotExist(std::string context) override { + resolver_->OnResourceDoesNotExist(std::move(context)); } private: @@ -235,8 +166,12 @@ class XdsResolver : public Resolver { class ClusterRef : public DualRefCounted { public: ClusterRef(RefCountedPtr resolver, - absl::string_view cluster_name) - : resolver_(std::move(resolver)), cluster_name_(cluster_name) {} + RefCountedPtr + cluster_subscription, + absl::string_view cluster_key) + : resolver_(std::move(resolver)), + cluster_subscription_(std::move(cluster_subscription)), + cluster_key_(cluster_key) {} void Orphan() override { XdsResolver* resolver_ptr = resolver_.get(); @@ -245,13 +180,16 @@ class XdsResolver : public Resolver { resolver->MaybeRemoveUnusedClusters(); }, DEBUG_LOCATION); + cluster_subscription_.reset(); } - const std::string& cluster_name() const { return cluster_name_; } + const std::string& cluster_key() const { return cluster_key_; } private: RefCountedPtr resolver_; - std::string cluster_name_; + RefCountedPtr + cluster_subscription_; + std::string cluster_key_; }; // A routing data including cluster refs and routes table held by the @@ -288,9 +226,7 @@ class XdsResolver : public Resolver { }; static absl::StatusOr> Create( - XdsResolver* resolver, - const std::vector& routes, - const Duration& default_max_stream_duration); + XdsResolver* resolver, const Duration& default_max_stream_duration); bool operator==(const RouteConfigData& other) const { return clusters_ == other.clusters_ && routes_ == other.routes_; @@ -322,9 +258,9 @@ class XdsResolver : public Resolver { return sc1->json_string() == sc2->json_string(); } - absl::Status AddRouteEntry(const XdsRouteConfigResource::Route& route, - const Duration& default_max_stream_duration, - XdsResolver* resolver); + absl::Status AddRouteEntry(XdsResolver* resolver, + const XdsRouteConfigResource::Route& route, + const Duration& default_max_stream_duration); std::map> clusters_; std::vector routes_; @@ -397,27 +333,31 @@ class XdsResolver : public Resolver { }; RefCountedPtr GetOrCreateClusterRef( - absl::string_view cluster_name) { - auto it = cluster_ref_map_.find(cluster_name); + absl::string_view cluster_key, absl::string_view cluster_name) { + auto it = cluster_ref_map_.find(cluster_key); if (it == cluster_ref_map_.end()) { - auto cluster = MakeRefCounted(RefAsSubclass(), - cluster_name); - cluster_ref_map_.emplace(cluster->cluster_name(), cluster->WeakRef()); + RefCountedPtr subscription; + if (!cluster_name.empty()) { + // The cluster ref will hold a subscription to ensure that the + // XdsDependencyManager stays subscribed to the CDS resource as + // long as the cluster ref exists. + subscription = dependency_mgr_->GetClusterSubscription(cluster_name); + } + auto cluster = MakeRefCounted( + RefAsSubclass(), std::move(subscription), cluster_key); + cluster_ref_map_.emplace(cluster->cluster_key(), cluster->WeakRef()); return cluster; } return it->second->Ref(); } - void OnListenerUpdate(std::shared_ptr listener); - void OnRouteConfigUpdate( - std::shared_ptr rds_update); + void OnUpdate(RefCountedPtr config); void OnError(absl::string_view context, absl::Status status); void OnResourceDoesNotExist(std::string context); absl::StatusOr> CreateServiceConfig(); void GenerateResult(); void MaybeRemoveUnusedClusters(); - uint64_t channel_id() const { return channel_id_; } std::shared_ptr work_serializer_; std::unique_ptr result_handler_; @@ -427,17 +367,10 @@ class XdsResolver : public Resolver { RefCountedPtr xds_client_; std::string lds_resource_name_; std::string data_plane_authority_; - uint64_t channel_id_; - - ListenerWatcher* listener_watcher_ = nullptr; - std::shared_ptr current_listener_; - - std::string route_config_name_; - RouteConfigWatcher* route_config_watcher_ = nullptr; - std::shared_ptr current_route_config_; - - const XdsRouteConfigResource::VirtualHost* current_virtual_host_ = nullptr; + const uint64_t channel_id_; + OrphanablePtr dependency_mgr_; + RefCountedPtr current_config_; std::map> cluster_ref_map_; }; @@ -470,19 +403,17 @@ class XdsResolver::RouteConfigData::RouteListIterator absl::StatusOr> XdsResolver::RouteConfigData::Create( - XdsResolver* resolver, - const std::vector& routes, - const Duration& default_max_stream_duration) { + XdsResolver* resolver, const Duration& default_max_stream_duration) { auto data = MakeRefCounted(); // Reserve the necessary entries up-front to avoid reallocation as we add // elements. This is necessary because the string_view in the entry's // weighted_cluster_state field points to the memory in the route field, so // moving the entry in a reallocation will cause the string_view to point to // invalid data. - data->routes_.reserve(routes.size()); - for (auto& route : routes) { + data->routes_.reserve(resolver->current_config_->virtual_host->routes.size()); + for (auto& route : resolver->current_config_->virtual_host->routes) { absl::Status status = - data->AddRouteEntry(route, default_max_stream_duration, resolver); + data->AddRouteEntry(resolver, route, default_max_stream_duration); if (!status.ok()) { return status; } @@ -555,12 +486,12 @@ XdsResolver::RouteConfigData::CreateMethodConfig( } // Handle xDS HTTP filters. const auto& hcm = absl::get( - resolver->current_listener_->listener); + resolver->current_config_->listener->listener); auto result = XdsRouting::GeneratePerHTTPFilterConfigs( static_cast(resolver->xds_client_->bootstrap()) .http_filter_registry(), - hcm.http_filters, *resolver->current_virtual_host_, route, cluster_weight, - resolver->args_); + hcm.http_filters, *resolver->current_config_->virtual_host, route, + cluster_weight, resolver->args_); if (!result.ok()) return result.status(); for (const auto& p : result->per_filter_configs) { fields.emplace_back(absl::StrCat(" \"", p.first, "\": [\n", @@ -585,19 +516,21 @@ XdsResolver::RouteConfigData::CreateMethodConfig( } absl::Status XdsResolver::RouteConfigData::AddRouteEntry( - const XdsRouteConfigResource::Route& route, - const Duration& default_max_stream_duration, XdsResolver* resolver) { + XdsResolver* resolver, const XdsRouteConfigResource::Route& route, + const Duration& default_max_stream_duration) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { gpr_log(GPR_INFO, "[xds_resolver %p] XdsConfigSelector %p: route: %s", resolver, this, route.ToString().c_str()); } routes_.emplace_back(route); auto* route_entry = &routes_.back(); - auto maybe_add_cluster = [&](absl::string_view cluster_name) { - if (clusters_.find(cluster_name) != clusters_.end()) return; - auto cluster_state = resolver->GetOrCreateClusterRef(cluster_name); - absl::string_view name = cluster_state->cluster_name(); - clusters_.emplace(name, std::move(cluster_state)); + auto maybe_add_cluster = [&](absl::string_view cluster_key, + absl::string_view cluster_name) { + if (clusters_.find(cluster_key) != clusters_.end()) return; + auto cluster_state = + resolver->GetOrCreateClusterRef(cluster_key, cluster_name); + absl::string_view key = cluster_state->cluster_key(); + clusters_.emplace(key, std::move(cluster_state)); }; auto* route_action = absl::get_if( &route_entry->route.action); @@ -618,8 +551,8 @@ absl::Status XdsResolver::RouteConfigData::AddRouteEntry( return result.status(); } route_entry->method_config = std::move(*result); - maybe_add_cluster( - absl::StrCat("cluster:", cluster_name.cluster_name)); + maybe_add_cluster(absl::StrCat("cluster:", cluster_name.cluster_name), + cluster_name.cluster_name); return absl::OkStatus(); }, // WeightedClusters @@ -640,7 +573,8 @@ absl::Status XdsResolver::RouteConfigData::AddRouteEntry( cluster_weight_state.cluster = weighted_cluster.name; route_entry->weighted_cluster_state.push_back( std::move(cluster_weight_state)); - maybe_add_cluster(absl::StrCat("cluster:", weighted_cluster.name)); + maybe_add_cluster(absl::StrCat("cluster:", weighted_cluster.name), + weighted_cluster.name); } return absl::OkStatus(); }, @@ -653,9 +587,11 @@ absl::Status XdsResolver::RouteConfigData::AddRouteEntry( return result.status(); } route_entry->method_config = std::move(*result); - maybe_add_cluster(absl::StrCat( - "cluster_specifier_plugin:", - cluster_specifier_plugin_name.cluster_specifier_plugin_name)); + maybe_add_cluster( + absl::StrCat( + "cluster_specifier_plugin:", + cluster_specifier_plugin_name.cluster_specifier_plugin_name), + /*subscription_name=*/""); return absl::OkStatus(); }); if (!status.ok()) { @@ -683,7 +619,7 @@ XdsResolver::XdsConfigSelector::XdsConfigSelector( static_cast(resolver_->xds_client_->bootstrap()) .http_filter_registry(); const auto& hcm = absl::get( - resolver_->current_listener_->listener); + resolver_->current_config_->listener->listener); for (const auto& http_filter : hcm.http_filters) { // Find filter. This is guaranteed to succeed, because it's checked // at config validation time in the XdsApi code. @@ -814,7 +750,7 @@ absl::Status XdsResolver::XdsConfigSelector::GetCallConfig( }, [&](const XdsRouteConfigResource::Route::RouteAction::HashPolicy:: ChannelId&) -> absl::optional { - return resolver_->channel_id(); + return resolver_->channel_id_; }); if (new_hash.has_value()) { // Rotating the old value prevents duplicate hash rules from cancelling @@ -840,7 +776,7 @@ absl::Status XdsResolver::XdsConfigSelector::GetCallConfig( parsed_method_configs); } args.service_config_call_data->SetCallAttribute( - args.arena->New(cluster->cluster_name())); + args.arena->New(cluster->cluster_key())); args.service_config_call_data->SetCallAttribute( args.arena->New(*hash)); args.service_config_call_data->SetCallAttribute( @@ -944,6 +880,9 @@ void XdsResolver::StartLocked() { return; } xds_client_ = std::move(*xds_client); + grpc_pollset_set_add_pollset_set(xds_client_->interested_parties(), + interested_parties_); + // Determine LDS resource name. std::string resource_name_fragment(absl::StripPrefix(uri_.path(), "/")); if (!uri_.authority().empty()) { // target_uri.authority is set case @@ -989,13 +928,11 @@ void XdsResolver::StartLocked() { gpr_log(GPR_INFO, "[xds_resolver %p] Started with lds_resource_name %s.", this, lds_resource_name_.c_str()); } - grpc_pollset_set_add_pollset_set( - static_cast(xds_client_.get())->interested_parties(), - interested_parties_); - auto watcher = MakeRefCounted(RefAsSubclass()); - listener_watcher_ = watcher.get(); - XdsListenerResourceType::StartWatch(xds_client_.get(), lds_resource_name_, - std::move(watcher)); + // Start watch for xDS config. + dependency_mgr_ = MakeOrphanable( + xds_client_, work_serializer_, + std::make_unique(RefAsSubclass()), + data_plane_authority_, lds_resource_name_, args_, interested_parties_); } void XdsResolver::ShutdownLocked() { @@ -1003,122 +940,20 @@ void XdsResolver::ShutdownLocked() { gpr_log(GPR_INFO, "[xds_resolver %p] shutting down", this); } if (xds_client_ != nullptr) { - if (listener_watcher_ != nullptr) { - XdsListenerResourceType::CancelWatch( - xds_client_.get(), lds_resource_name_, listener_watcher_, - /*delay_unsubscription=*/false); - } - if (route_config_watcher_ != nullptr) { - XdsRouteConfigResourceType::CancelWatch( - xds_client_.get(), route_config_name_, route_config_watcher_, - /*delay_unsubscription=*/false); - } - grpc_pollset_set_del_pollset_set( - static_cast(xds_client_.get())->interested_parties(), - interested_parties_); + dependency_mgr_.reset(); + grpc_pollset_set_del_pollset_set(xds_client_->interested_parties(), + interested_parties_); xds_client_.reset(DEBUG_LOCATION, "xds resolver"); } } -void XdsResolver::OnListenerUpdate( - std::shared_ptr listener) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_resolver %p] received updated listener data", this); - } - if (xds_client_ == nullptr) return; - const auto* hcm = absl::get_if( - &listener->listener); - if (hcm == nullptr) { - return OnError(lds_resource_name_, - absl::UnavailableError("not an API listener")); - } - current_listener_ = std::move(listener); - Match( - hcm->route_config, - // RDS resource name - [&](const std::string& rds_name) { - // If the RDS name changed, update the RDS watcher. - // Note that this will be true on the initial update, because - // route_config_name_ will be empty. - if (route_config_name_ != rds_name) { - // If we already had a watch (i.e., if the previous config had - // a different RDS name), stop the previous watch. - // There will be no previous watch if either (a) this is the - // initial resource update or (b) the previous Listener had an - // inlined RouteConfig. - if (route_config_watcher_ != nullptr) { - XdsRouteConfigResourceType::CancelWatch( - xds_client_.get(), route_config_name_, route_config_watcher_, - /*delay_unsubscription=*/true); - route_config_watcher_ = nullptr; - } - // Start watch for the new RDS resource name. - route_config_name_ = rds_name; - auto watcher = - MakeRefCounted(RefAsSubclass()); - route_config_watcher_ = watcher.get(); - XdsRouteConfigResourceType::StartWatch( - xds_client_.get(), route_config_name_, std::move(watcher)); - } else { - // RDS resource name has not changed, so no watch needs to be - // updated, but we still need to propagate any changes in the - // HCM config (e.g., the list of HTTP filters). - GenerateResult(); - } - }, - // inlined RouteConfig - [&](const std::shared_ptr& route_config) { - // If the previous update specified an RDS resource instead of - // having an inlined RouteConfig, we need to cancel the RDS watch. - if (route_config_watcher_ != nullptr) { - XdsRouteConfigResourceType::CancelWatch( - xds_client_.get(), route_config_name_, route_config_watcher_); - route_config_watcher_ = nullptr; - route_config_name_.clear(); - } - OnRouteConfigUpdate(route_config); - }); -} - -class VirtualHostListIterator : public XdsRouting::VirtualHostListIterator { - public: - explicit VirtualHostListIterator( - const std::vector* virtual_hosts) - : virtual_hosts_(virtual_hosts) {} - - size_t Size() const override { return virtual_hosts_->size(); } - - const std::vector& GetDomainsForVirtualHost( - size_t index) const override { - return (*virtual_hosts_)[index].domains; - } - - private: - const std::vector* virtual_hosts_; -}; - -void XdsResolver::OnRouteConfigUpdate( - std::shared_ptr rds_update) { +void XdsResolver::OnUpdate( + RefCountedPtr config) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_resolver_trace)) { - gpr_log(GPR_INFO, "[xds_resolver %p] received updated route config", this); + gpr_log(GPR_INFO, "[xds_resolver %p] received updated xDS config", this); } if (xds_client_ == nullptr) return; - // Find the relevant VirtualHost from the RouteConfiguration. - auto vhost_index = XdsRouting::FindVirtualHostForDomain( - VirtualHostListIterator(&rds_update->virtual_hosts), - data_plane_authority_); - if (!vhost_index.has_value()) { - OnError( - route_config_name_.empty() ? lds_resource_name_ : route_config_name_, - absl::UnavailableError(absl::StrCat("could not find VirtualHost for ", - data_plane_authority_, - " in RouteConfiguration"))); - return; - } - // Save the virtual host in the resolver. - current_route_config_ = std::move(rds_update); - current_virtual_host_ = ¤t_route_config_->virtual_hosts[*vhost_index]; - // Send a new result to the channel. + current_config_ = std::move(config); GenerateResult(); } @@ -1142,7 +977,7 @@ void XdsResolver::OnResourceDoesNotExist(std::string context) { "update and returning empty service config", this); if (xds_client_ == nullptr) return; - current_virtual_host_ = nullptr; + current_config_.reset(); Result result; result.addresses.emplace(); result.service_config = ServiceConfigImpl::Create(args_, "{}"); @@ -1163,7 +998,7 @@ XdsResolver::CreateServiceConfig() { " \"childPolicy\": %s\n" " }", cluster.first, - current_route_config_->cluster_specifier_plugin_map.at( + current_config_->route_config->cluster_specifier_plugin_map.at( std::string(child_name)))); } else { absl::ConsumePrefix(&child_name, "cluster:"); @@ -1195,13 +1030,13 @@ XdsResolver::CreateServiceConfig() { } void XdsResolver::GenerateResult() { - if (current_virtual_host_ == nullptr) return; + if (xds_client_ == nullptr || current_config_ == nullptr) return; // First create XdsConfigSelector, which may add new entries to the cluster - // state map, and then CreateServiceConfig for LB policies. + // state map. const auto& hcm = absl::get( - current_listener_->listener); - auto route_config_data = RouteConfigData::Create( - this, current_virtual_host_->routes, hcm.http_max_stream_duration); + current_config_->listener->listener); + auto route_config_data = + RouteConfigData::Create(this, hcm.http_max_stream_duration); if (!route_config_data.ok()) { OnError("could not create ConfigSelector", absl::UnavailableError(route_config_data.status().message())); @@ -1209,6 +1044,7 @@ void XdsResolver::GenerateResult() { } auto config_selector = MakeRefCounted( RefAsSubclass(), std::move(*route_config_data)); + // Now create the service config. Result result; result.addresses.emplace(); result.service_config = CreateServiceConfig(); @@ -1220,7 +1056,9 @@ void XdsResolver::GenerateResult() { } result.args = args_.SetObject(xds_client_.Ref(DEBUG_LOCATION, "xds resolver result")) - .SetObject(config_selector); + .SetObject(config_selector) + .SetObject(current_config_) + .SetObject(dependency_mgr_->Ref()); result_handler_->ReportResult(std::move(result)); } @@ -1235,10 +1073,7 @@ void XdsResolver::MaybeRemoveUnusedClusters() { it = cluster_ref_map_.erase(it); } } - if (update_needed && xds_client_ != nullptr) { - // Send a new result to the channel. - GenerateResult(); - } + if (update_needed) GenerateResult(); } // diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h similarity index 93% rename from src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h rename to src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h index 93c48ac0d7d..f8b8c838f8e 100644 --- a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h @@ -14,8 +14,8 @@ // limitations under the License. // -#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_H -#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_H +#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_ATTRIBUTES_H +#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_ATTRIBUTES_H #include @@ -49,13 +49,14 @@ class XdsRouteStateAttribute : public ServiceConfigCallData::CallAttributeInterface { public: static UniqueTypeName TypeName() { - static UniqueTypeName::Factory factory("xds_cluster_lb_data"); + static UniqueTypeName::Factory factory("xds_route_state"); return factory.Create(); } virtual bool HasClusterForRoute(absl::string_view cluster_name) const = 0; UniqueTypeName type() const override { return TypeName(); } }; + } // namespace grpc_core -#endif // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_H +#endif // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_ATTRIBUTES_H diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc new file mode 100644 index 00000000000..a56990484bb --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc @@ -0,0 +1,25 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h" + +namespace grpc_core { + +TraceFlag grpc_xds_resolver_trace(false, "xds_resolver"); + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h new file mode 100644 index 00000000000..63e3396639a --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h @@ -0,0 +1,30 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_TRACE_H +#define GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_TRACE_H + +#include + +#include "src/core/lib/debug/trace.h" + +namespace grpc_core { + +extern TraceFlag grpc_xds_resolver_trace; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_XDS_XDS_RESOLVER_TRACE_H diff --git a/src/core/ext/filters/stateful_session/stateful_session_filter.cc b/src/core/ext/filters/stateful_session/stateful_session_filter.cc index 4ed600937c0..f0c3afb2d0e 100644 --- a/src/core/ext/filters/stateful_session/stateful_session_filter.cc +++ b/src/core/ext/filters/stateful_session/stateful_session_filter.cc @@ -38,7 +38,7 @@ #include -#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h" +#include "src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h" #include "src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" diff --git a/src/core/ext/xds/xds_bootstrap.h b/src/core/ext/xds/xds_bootstrap.h index 23ba5ea8e2d..cc38cbb9b41 100644 --- a/src/core/ext/xds/xds_bootstrap.h +++ b/src/core/ext/xds/xds_bootstrap.h @@ -53,6 +53,9 @@ class XdsBootstrap { friend bool operator==(const XdsServer& a, const XdsServer& b) { return a.Equals(b); } + friend bool operator!=(const XdsServer& a, const XdsServer& b) { + return !a.Equals(b); + } }; class Authority { diff --git a/src/core/ext/xds/xds_certificate_provider.cc b/src/core/ext/xds/xds_certificate_provider.cc index 439921c648a..b97f0acb05f 100644 --- a/src/core/ext/xds/xds_certificate_provider.cc +++ b/src/core/ext/xds/xds_certificate_provider.cc @@ -43,16 +43,15 @@ class RootCertificatesWatcher // presently, the watcher is immediately deleted when // CancelTlsCertificatesWatch() is called, but that can potentially change in // the future. - RootCertificatesWatcher( - RefCountedPtr parent, - std::string cert_name) - : parent_(std::move(parent)), cert_name_(std::move(cert_name)) {} + explicit RootCertificatesWatcher( + RefCountedPtr parent) + : parent_(std::move(parent)) {} void OnCertificatesChanged(absl::optional root_certs, absl::optional /* key_cert_pairs */) override { if (root_certs.has_value()) { - parent_->SetKeyMaterials(cert_name_, std::string(root_certs.value()), + parent_->SetKeyMaterials("", std::string(root_certs.value()), absl::nullopt); } } @@ -60,14 +59,13 @@ class RootCertificatesWatcher void OnError(grpc_error_handle root_cert_error, grpc_error_handle /*identity_cert_error*/) override { if (!root_cert_error.ok()) { - parent_->SetErrorForCert(cert_name_, root_cert_error /* pass the ref */, + parent_->SetErrorForCert("", root_cert_error /* pass the ref */, absl::nullopt); } } private: RefCountedPtr parent_; - std::string cert_name_; }; class IdentityCertificatesWatcher @@ -78,300 +76,142 @@ class IdentityCertificatesWatcher // presently, the watcher is immediately deleted when // CancelTlsCertificatesWatch() is called, but that can potentially change in // the future. - IdentityCertificatesWatcher( - RefCountedPtr parent, - std::string cert_name) - : parent_(std::move(parent)), cert_name_(std::move(cert_name)) {} + explicit IdentityCertificatesWatcher( + RefCountedPtr parent) + : parent_(std::move(parent)) {} void OnCertificatesChanged( absl::optional /* root_certs */, absl::optional key_cert_pairs) override { if (key_cert_pairs.has_value()) { - parent_->SetKeyMaterials(cert_name_, absl::nullopt, key_cert_pairs); + parent_->SetKeyMaterials("", absl::nullopt, key_cert_pairs); } } void OnError(grpc_error_handle /*root_cert_error*/, grpc_error_handle identity_cert_error) override { if (!identity_cert_error.ok()) { - parent_->SetErrorForCert(cert_name_, absl::nullopt, + parent_->SetErrorForCert("", absl::nullopt, identity_cert_error /* pass the ref */); } } private: RefCountedPtr parent_; - std::string cert_name_; }; } // namespace // -// XdsCertificateProvider::ClusterCertificateState +// XdsCertificateProvider // -XdsCertificateProvider::ClusterCertificateState::~ClusterCertificateState() { - if (root_cert_watcher_ != nullptr) { - root_cert_distributor_->CancelTlsCertificatesWatch(root_cert_watcher_); - } - if (identity_cert_watcher_ != nullptr) { - identity_cert_distributor_->CancelTlsCertificatesWatch( - identity_cert_watcher_); - } +XdsCertificateProvider::XdsCertificateProvider( + RefCountedPtr root_cert_provider, + absl::string_view root_cert_name, + RefCountedPtr identity_cert_provider, + absl::string_view identity_cert_name, + std::vector san_matchers) + : distributor_(MakeRefCounted()), + root_cert_provider_(std::move(root_cert_provider)), + root_cert_name_(root_cert_name), + identity_cert_provider_(std::move(identity_cert_provider)), + identity_cert_name_(identity_cert_name), + san_matchers_(std::move(san_matchers)), + require_client_certificate_(false) { + distributor_->SetWatchStatusCallback( + absl::bind_front(&XdsCertificateProvider::WatchStatusCallback, this)); } -bool XdsCertificateProvider::ClusterCertificateState::IsSafeToRemove() const { - return !watching_root_certs_ && !watching_identity_certs_ && - root_cert_distributor_ == nullptr && - identity_cert_distributor_ == nullptr; +XdsCertificateProvider::XdsCertificateProvider( + RefCountedPtr root_cert_provider, + absl::string_view root_cert_name, + RefCountedPtr identity_cert_provider, + absl::string_view identity_cert_name, bool require_client_certificate) + : distributor_(MakeRefCounted()), + root_cert_provider_(std::move(root_cert_provider)), + root_cert_name_(root_cert_name), + identity_cert_provider_(std::move(identity_cert_provider)), + identity_cert_name_(identity_cert_name), + require_client_certificate_(require_client_certificate) { + distributor_->SetWatchStatusCallback( + absl::bind_front(&XdsCertificateProvider::WatchStatusCallback, this)); } -void XdsCertificateProvider::ClusterCertificateState:: - UpdateRootCertNameAndDistributor( - const std::string& cert_name, absl::string_view root_cert_name, - RefCountedPtr root_cert_distributor) { - if (root_cert_name_ == root_cert_name && - root_cert_distributor_ == root_cert_distributor) { - return; - } - root_cert_name_ = std::string(root_cert_name); - if (watching_root_certs_) { - // The root certificates are being watched. Swap out the watcher. - if (root_cert_distributor_ != nullptr) { - root_cert_distributor_->CancelTlsCertificatesWatch(root_cert_watcher_); - } - if (root_cert_distributor != nullptr) { - UpdateRootCertWatcher(cert_name, root_cert_distributor.get()); - } else { - root_cert_watcher_ = nullptr; - xds_certificate_provider_->distributor_->SetErrorForCert( - "", +XdsCertificateProvider::~XdsCertificateProvider() { + distributor_->SetWatchStatusCallback(nullptr); +} + +UniqueTypeName XdsCertificateProvider::type() const { + static UniqueTypeName::Factory kFactory("Xds"); + return kFactory.Create(); +} + +void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, + bool root_being_watched, + bool identity_being_watched) { + if (!cert_name.empty()) { + if (root_being_watched) { + distributor_->SetErrorForCert( + cert_name, GRPC_ERROR_CREATE( "No certificate provider available for root certificates"), absl::nullopt); } - } - // Swap out the root certificate distributor - root_cert_distributor_ = std::move(root_cert_distributor); -} - -void XdsCertificateProvider::ClusterCertificateState:: - UpdateIdentityCertNameAndDistributor( - const std::string& cert_name, absl::string_view identity_cert_name, - RefCountedPtr - identity_cert_distributor) { - if (identity_cert_name_ == identity_cert_name && - identity_cert_distributor_ == identity_cert_distributor) { - return; - } - identity_cert_name_ = std::string(identity_cert_name); - if (watching_identity_certs_) { - // The identity certificates are being watched. Swap out the watcher. - if (identity_cert_distributor_ != nullptr) { - identity_cert_distributor_->CancelTlsCertificatesWatch( - identity_cert_watcher_); - } - if (identity_cert_distributor != nullptr) { - UpdateIdentityCertWatcher(cert_name, identity_cert_distributor.get()); - } else { - identity_cert_watcher_ = nullptr; - xds_certificate_provider_->distributor_->SetErrorForCert( - "", absl::nullopt, + if (identity_being_watched) { + distributor_->SetErrorForCert( + cert_name, absl::nullopt, GRPC_ERROR_CREATE( "No certificate provider available for identity certificates")); } + return; } - // Swap out the identity certificate distributor - identity_cert_distributor_ = std::move(identity_cert_distributor); -} - -void XdsCertificateProvider::ClusterCertificateState::UpdateRootCertWatcher( - const std::string& cert_name, - grpc_tls_certificate_distributor* root_cert_distributor) { - auto watcher = std::make_unique( - xds_certificate_provider_->distributor_, cert_name); - root_cert_watcher_ = watcher.get(); - root_cert_distributor->WatchTlsCertificates(std::move(watcher), - root_cert_name_, absl::nullopt); -} - -void XdsCertificateProvider::ClusterCertificateState::UpdateIdentityCertWatcher( - const std::string& cert_name, - grpc_tls_certificate_distributor* identity_cert_distributor) { - auto watcher = std::make_unique( - xds_certificate_provider_->distributor_, cert_name); - identity_cert_watcher_ = watcher.get(); - identity_cert_distributor->WatchTlsCertificates( - std::move(watcher), absl::nullopt, identity_cert_name_); -} - -void XdsCertificateProvider::ClusterCertificateState::WatchStatusCallback( - const std::string& cert_name, bool root_being_watched, - bool identity_being_watched) { // We aren't specially handling the case where root_cert_distributor is same // as identity_cert_distributor. Always using two separate watchers // irrespective of the fact results in a straightforward design, and using a // single watcher does not seem to provide any benefit other than cutting down // on the number of callbacks. - if (root_being_watched && !watching_root_certs_) { - // We need to start watching root certs. - watching_root_certs_ = true; - if (root_cert_distributor_ == nullptr) { - xds_certificate_provider_->distributor_->SetErrorForCert( + if (root_being_watched && root_cert_watcher_ == nullptr) { + // Start watching root cert. + if (root_cert_provider_ == nullptr) { + distributor_->SetErrorForCert( cert_name, GRPC_ERROR_CREATE( "No certificate provider available for root certificates"), absl::nullopt); } else { - UpdateRootCertWatcher(cert_name, root_cert_distributor_.get()); - } - } else if (!root_being_watched && watching_root_certs_) { - // We need to cancel root certs watch. - watching_root_certs_ = false; - if (root_cert_distributor_ != nullptr) { - root_cert_distributor_->CancelTlsCertificatesWatch(root_cert_watcher_); - root_cert_watcher_ = nullptr; + auto watcher = std::make_unique(distributor_); + root_cert_watcher_ = watcher.get(); + root_cert_provider_->distributor()->WatchTlsCertificates( + std::move(watcher), root_cert_name_, absl::nullopt); } - GPR_ASSERT(root_cert_watcher_ == nullptr); - } - if (identity_being_watched && !watching_identity_certs_) { - watching_identity_certs_ = true; - if (identity_cert_distributor_ == nullptr) { - xds_certificate_provider_->distributor_->SetErrorForCert( + } else if (!root_being_watched && root_cert_watcher_ != nullptr) { + // Cancel root cert watch. + GPR_ASSERT(root_cert_provider_ != nullptr); + root_cert_provider_->distributor()->CancelTlsCertificatesWatch( + root_cert_watcher_); + root_cert_watcher_ = nullptr; + } + if (identity_being_watched && identity_cert_watcher_ == nullptr) { + // Start watching identity cert. + if (identity_cert_provider_ == nullptr) { + distributor_->SetErrorForCert( cert_name, absl::nullopt, GRPC_ERROR_CREATE( "No certificate provider available for identity certificates")); } else { - UpdateIdentityCertWatcher(cert_name, identity_cert_distributor_.get()); - } - } else if (!identity_being_watched && watching_identity_certs_) { - watching_identity_certs_ = false; - if (identity_cert_distributor_ != nullptr) { - identity_cert_distributor_->CancelTlsCertificatesWatch( - identity_cert_watcher_); - identity_cert_watcher_ = nullptr; + auto watcher = + std::make_unique(distributor_); + identity_cert_watcher_ = watcher.get(); + identity_cert_provider_->distributor()->WatchTlsCertificates( + std::move(watcher), absl::nullopt, identity_cert_name_); } - GPR_ASSERT(identity_cert_watcher_ == nullptr); - } -} - -// -// XdsCertificateProvider -// - -XdsCertificateProvider::XdsCertificateProvider() - : distributor_(MakeRefCounted()) { - distributor_->SetWatchStatusCallback( - absl::bind_front(&XdsCertificateProvider::WatchStatusCallback, this)); -} - -XdsCertificateProvider::~XdsCertificateProvider() { - distributor_->SetWatchStatusCallback(nullptr); -} - -UniqueTypeName XdsCertificateProvider::type() const { - static UniqueTypeName::Factory kFactory("Xds"); - return kFactory.Create(); -} - -bool XdsCertificateProvider::ProvidesRootCerts(const std::string& cert_name) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) return false; - return it->second->ProvidesRootCerts(); -} - -void XdsCertificateProvider::UpdateRootCertNameAndDistributor( - const std::string& cert_name, absl::string_view root_cert_name, - RefCountedPtr root_cert_distributor) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) { - it = - certificate_state_map_ - .emplace(cert_name, std::make_unique(this)) - .first; - } - it->second->UpdateRootCertNameAndDistributor(cert_name, root_cert_name, - root_cert_distributor); - // Delete unused entries. - if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); -} - -bool XdsCertificateProvider::ProvidesIdentityCerts( - const std::string& cert_name) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) return false; - return it->second->ProvidesIdentityCerts(); -} - -void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( - const std::string& cert_name, absl::string_view identity_cert_name, - RefCountedPtr identity_cert_distributor) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) { - it = - certificate_state_map_ - .emplace(cert_name, std::make_unique(this)) - .first; - } - it->second->UpdateIdentityCertNameAndDistributor( - cert_name, identity_cert_name, identity_cert_distributor); - // Delete unused entries. - if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); -} - -bool XdsCertificateProvider::GetRequireClientCertificate( - const std::string& cert_name) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) return false; - return it->second->require_client_certificate(); -} - -void XdsCertificateProvider::UpdateRequireClientCertificate( - const std::string& cert_name, bool require_client_certificate) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) return; - it->second->set_require_client_certificate(require_client_certificate); -} - -std::vector XdsCertificateProvider::GetSanMatchers( - const std::string& cluster) { - MutexLock lock(&san_matchers_mu_); - auto it = san_matcher_map_.find(cluster); - if (it == san_matcher_map_.end()) return {}; - return it->second; -} - -void XdsCertificateProvider::UpdateSubjectAlternativeNameMatchers( - const std::string& cluster, std::vector matchers) { - MutexLock lock(&san_matchers_mu_); - if (matchers.empty()) { - san_matcher_map_.erase(cluster); - } else { - san_matcher_map_[cluster] = std::move(matchers); - } -} - -void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, - bool root_being_watched, - bool identity_being_watched) { - MutexLock lock(&mu_); - auto it = certificate_state_map_.find(cert_name); - if (it == certificate_state_map_.end()) { - it = - certificate_state_map_ - .emplace(cert_name, std::make_unique(this)) - .first; + } else if (!identity_being_watched && identity_cert_watcher_ != nullptr) { + GPR_ASSERT(identity_cert_provider_ != nullptr); + identity_cert_provider_->distributor()->CancelTlsCertificatesWatch( + identity_cert_watcher_); + identity_cert_watcher_ = nullptr; } - it->second->WatchStatusCallback(cert_name, root_being_watched, - identity_being_watched); - // Delete unused entries. - if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); } } // namespace grpc_core diff --git a/src/core/ext/xds/xds_certificate_provider.h b/src/core/ext/xds/xds_certificate_provider.h index 44eb2421d89..3d6dca2f59b 100644 --- a/src/core/ext/xds/xds_certificate_provider.h +++ b/src/core/ext/xds/xds_certificate_provider.h @@ -44,7 +44,21 @@ namespace grpc_core { class XdsCertificateProvider : public grpc_tls_certificate_provider { public: - XdsCertificateProvider(); + // ctor for client side + XdsCertificateProvider( + RefCountedPtr root_cert_provider, + absl::string_view root_cert_name, + RefCountedPtr identity_cert_provider, + absl::string_view identity_cert_name, + std::vector san_matchers); + + // ctor for server side + XdsCertificateProvider( + RefCountedPtr root_cert_provider, + absl::string_view root_cert_name, + RefCountedPtr identity_cert_provider, + absl::string_view identity_cert_name, bool require_client_certificate); + ~XdsCertificateProvider() override; RefCountedPtr distributor() const override { @@ -53,94 +67,27 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { UniqueTypeName type() const override; - bool ProvidesRootCerts(const std::string& cert_name); - void UpdateRootCertNameAndDistributor( - const std::string& cert_name, absl::string_view root_cert_name, - RefCountedPtr root_cert_distributor); - - bool ProvidesIdentityCerts(const std::string& cert_name); - void UpdateIdentityCertNameAndDistributor( - const std::string& cert_name, absl::string_view identity_cert_name, - RefCountedPtr - identity_cert_distributor); - - bool GetRequireClientCertificate(const std::string& cert_name); - // Updating \a require_client_certificate for a non-existing \a cert_name has - // no effect. - void UpdateRequireClientCertificate(const std::string& cert_name, - bool require_client_certificate); - - std::vector GetSanMatchers(const std::string& cluster); - void UpdateSubjectAlternativeNameMatchers( - const std::string& cluster, std::vector matchers); + bool ProvidesRootCerts() const { return root_cert_provider_ != nullptr; } + bool ProvidesIdentityCerts() const { + return identity_cert_provider_ != nullptr; + } + bool require_client_certificate() const { + return require_client_certificate_; + } + const std::vector& san_matchers() const { + return san_matchers_; + } static absl::string_view ChannelArgName() { return "grpc.internal.xds_certificate_provider"; } static int ChannelArgsCompare(const XdsCertificateProvider* a, const XdsCertificateProvider* b) { + if (a == nullptr || b == nullptr) return QsortCompare(a, b); return a->Compare(b); } private: - class ClusterCertificateState { - public: - explicit ClusterCertificateState( - XdsCertificateProvider* xds_certificate_provider) - : xds_certificate_provider_(xds_certificate_provider) {} - - ~ClusterCertificateState(); - - // Returns true if the certs aren't being watched and there are no - // distributors configured. - bool IsSafeToRemove() const; - - bool ProvidesRootCerts() const { return root_cert_distributor_ != nullptr; } - bool ProvidesIdentityCerts() const { - return identity_cert_distributor_ != nullptr; - } - - void UpdateRootCertNameAndDistributor( - const std::string& cert_name, absl::string_view root_cert_name, - RefCountedPtr root_cert_distributor); - void UpdateIdentityCertNameAndDistributor( - const std::string& cert_name, absl::string_view identity_cert_name, - RefCountedPtr - identity_cert_distributor); - - void UpdateRootCertWatcher( - const std::string& cert_name, - grpc_tls_certificate_distributor* root_cert_distributor); - void UpdateIdentityCertWatcher( - const std::string& cert_name, - grpc_tls_certificate_distributor* identity_cert_distributor); - - bool require_client_certificate() const { - return require_client_certificate_; - } - void set_require_client_certificate(bool require_client_certificate) { - require_client_certificate_ = require_client_certificate; - } - - void WatchStatusCallback(const std::string& cert_name, - bool root_being_watched, - bool identity_being_watched); - - private: - XdsCertificateProvider* xds_certificate_provider_; - bool watching_root_certs_ = false; - bool watching_identity_certs_ = false; - std::string root_cert_name_; - std::string identity_cert_name_; - RefCountedPtr root_cert_distributor_; - RefCountedPtr identity_cert_distributor_; - grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* - root_cert_watcher_ = nullptr; - grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* - identity_cert_watcher_ = nullptr; - bool require_client_certificate_ = false; - }; - int CompareImpl(const grpc_tls_certificate_provider* other) const override { // TODO(yashykt): Maybe do something better here. return QsortCompare(static_cast(this), @@ -151,22 +98,17 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { bool identity_being_watched); RefCountedPtr distributor_; - - Mutex mu_; - std::map> - certificate_state_map_ ABSL_GUARDED_BY(mu_); - - // Use a separate mutex for san_matchers_ to avoid deadlocks since - // san_matchers_ needs to be accessed when a handshake is being done and we - // run into a possible deadlock scenario if using the same mutex. The mutex - // deadlock cycle is formed as - - // WatchStatusCallback() -> SetKeyMaterials() -> - // TlsChannelSecurityConnector::TlsChannelCertificateWatcher::OnCertificatesChanged() - // -> HandshakeManager::Add() -> SecurityHandshaker::DoHandshake() -> - // subject_alternative_names_matchers() - Mutex san_matchers_mu_; - std::map> - san_matcher_map_ ABSL_GUARDED_BY(san_matchers_mu_); + RefCountedPtr root_cert_provider_; + std::string root_cert_name_; + RefCountedPtr identity_cert_provider_; + std::string identity_cert_name_; + std::vector san_matchers_; + bool require_client_certificate_ = false; + + grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* + root_cert_watcher_ = nullptr; + grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* + identity_cert_watcher_ = nullptr; }; } // namespace grpc_core diff --git a/src/core/ext/xds/xds_cluster.cc b/src/core/ext/xds/xds_cluster.cc index 8b44720c155..2386a43a521 100644 --- a/src/core/ext/xds/xds_cluster.cc +++ b/src/core/ext/xds/xds_cluster.cc @@ -107,15 +107,8 @@ std::string XdsClusterResource::ToString() const { } contents.push_back( absl::StrCat("max_concurrent_requests=", max_concurrent_requests)); - if (!override_host_statuses.empty()) { - std::vector statuses; - statuses.reserve(override_host_statuses.size()); - for (const auto& status : override_host_statuses) { - statuses.push_back(status.ToString()); - } - contents.push_back(absl::StrCat("override_host_statuses={", - absl::StrJoin(statuses, ", "), "}")); - } + contents.push_back(absl::StrCat("override_host_statuses=", + override_host_statuses.ToString())); return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); } @@ -625,6 +618,7 @@ absl::StatusOr> CdsResourceParse( // Validate override host status. const auto* common_lb_config = envoy_config_cluster_v3_Cluster_common_lb_config(cluster); + bool override_host_status_found = false; if (common_lb_config != nullptr) { ValidationErrors::ScopedField field(&errors, ".common_lb_config"); const auto* override_host_status = @@ -638,11 +632,19 @@ absl::StatusOr> CdsResourceParse( for (size_t i = 0; i < size; ++i) { auto status = XdsHealthStatus::FromUpb(statuses[i]); if (status.has_value()) { - cds_update->override_host_statuses.insert(*status); + cds_update->override_host_statuses.Add(*status); } } + override_host_status_found = true; } } + // If the field is not set, we default to [UNKNOWN, HEALTHY]. + if (!override_host_status_found) { + cds_update->override_host_statuses.Add( + XdsHealthStatus(XdsHealthStatus::kUnknown)); + cds_update->override_host_statuses.Add( + XdsHealthStatus(XdsHealthStatus::kHealthy)); + } // Return result. if (!errors.ok()) { return errors.status(absl::StatusCode::kInvalidArgument, diff --git a/src/core/ext/xds/xds_cluster.h b/src/core/ext/xds/xds_cluster.h index efdc606f8c4..4a4749b6bf3 100644 --- a/src/core/ext/xds/xds_cluster.h +++ b/src/core/ext/xds/xds_cluster.h @@ -95,7 +95,7 @@ struct XdsClusterResource : public XdsResourceType::ResourceData { absl::optional outlier_detection; - std::set override_host_statuses; + XdsHealthStatusSet override_host_statuses; bool operator==(const XdsClusterResource& other) const { return type == other.type && lb_policy_config == other.lb_policy_config && diff --git a/src/core/ext/xds/xds_cluster_specifier_plugin.cc b/src/core/ext/xds/xds_cluster_specifier_plugin.cc index f5f4b478bb9..4df11a40088 100644 --- a/src/core/ext/xds/xds_cluster_specifier_plugin.cc +++ b/src/core/ext/xds/xds_cluster_specifier_plugin.cc @@ -99,7 +99,10 @@ Json XdsRouteLookupClusterSpecifierPlugin::GenerateLoadBalancingPolicyConfig( {"routeLookupConfig", std::move(*json)}, {"childPolicy", Json::FromArray({ - Json::FromObject({{"cds_experimental", Json::FromObject({})}}), + Json::FromObject({{"cds_experimental", + Json::FromObject({ + {"isDynamic", Json::FromBool(true)}, + })}}), })}, {"childPolicyConfigTargetFieldName", Json::FromString("cluster")}, })}})}); diff --git a/src/core/ext/xds/xds_health_status.cc b/src/core/ext/xds/xds_health_status.cc index 57f719f26cd..ad504ad91d0 100644 --- a/src/core/ext/xds/xds_health_status.cc +++ b/src/core/ext/xds/xds_health_status.cc @@ -20,6 +20,8 @@ #include +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" #include "envoy/config/core/v3/health_check.upb.h" namespace grpc_core { @@ -58,8 +60,16 @@ const char* XdsHealthStatus::ToString() const { } } -bool operator<(const XdsHealthStatus& hs1, const XdsHealthStatus& hs2) { - return hs1.status() < hs2.status(); +std::string XdsHealthStatusSet::ToString() const { + std::vector set; + set.reserve(3); + for (const auto& status : + {XdsHealthStatus::kUnknown, XdsHealthStatus::kHealthy, + XdsHealthStatus::kDraining}) { + const XdsHealthStatus health_status(status); + if (Contains(health_status)) set.push_back(health_status.ToString()); + } + return absl::StrCat("{", absl::StrJoin(set, ", "), "}"); } } // namespace grpc_core diff --git a/src/core/ext/xds/xds_health_status.h b/src/core/ext/xds/xds_health_status.h index c94b2ce6e78..73fbc3df4a3 100644 --- a/src/core/ext/xds/xds_health_status.h +++ b/src/core/ext/xds/xds_health_status.h @@ -70,6 +70,8 @@ class XdsHealthStatusSet { return status_mask_ == other.status_mask_; } + bool Empty() const { return status_mask_ == 0; } + void Clear() { status_mask_ = 0; } void Add(XdsHealthStatus status) { status_mask_ |= (0x1 << status.status()); } @@ -78,12 +80,12 @@ class XdsHealthStatusSet { return status_mask_ & (0x1 << status.status()); } + std::string ToString() const; + private: int status_mask_ = 0; }; -bool operator<(const XdsHealthStatus& hs1, const XdsHealthStatus& hs2); - } // namespace grpc_core #endif // GRPC_SRC_CORE_EXT_XDS_XDS_HEALTH_STATUS_H diff --git a/src/core/ext/xds/xds_route_config.cc b/src/core/ext/xds/xds_route_config.cc index efe7b802a34..4f349682fab 100644 --- a/src/core/ext/xds/xds_route_config.cc +++ b/src/core/ext/xds/xds_route_config.cc @@ -288,33 +288,44 @@ std::string XdsRouteConfigResource::Route::ToString() const { return absl::StrJoin(contents, "\n"); } +// +// XdsRouteConfigResource::Route +// + +std::string XdsRouteConfigResource::VirtualHost::ToString() const { + std::vector parts; + parts.push_back( + absl::StrCat("vhost={\n" + " domains=[", + absl::StrJoin(domains, ", "), + "]\n" + " routes=[\n")); + for (const XdsRouteConfigResource::Route& route : routes) { + parts.push_back(" {\n"); + parts.push_back(route.ToString()); + parts.push_back("\n }\n"); + } + parts.push_back(" ]\n"); + parts.push_back(" typed_per_filter_config={\n"); + for (const auto& p : typed_per_filter_config) { + const std::string& name = p.first; + const auto& config = p.second; + parts.push_back(absl::StrCat(" ", name, "=", config.ToString(), "\n")); + } + parts.push_back(" }\n"); + parts.push_back("}\n"); + return absl::StrJoin(parts, ""); +} + // // XdsRouteConfigResource // std::string XdsRouteConfigResource::ToString() const { std::vector parts; + parts.reserve(virtual_hosts.size()); for (const VirtualHost& vhost : virtual_hosts) { - parts.push_back( - absl::StrCat("vhost={\n" - " domains=[", - absl::StrJoin(vhost.domains, ", "), - "]\n" - " routes=[\n")); - for (const XdsRouteConfigResource::Route& route : vhost.routes) { - parts.push_back(" {\n"); - parts.push_back(route.ToString()); - parts.push_back("\n }\n"); - } - parts.push_back(" ]\n"); - parts.push_back(" typed_per_filter_config={\n"); - for (const auto& p : vhost.typed_per_filter_config) { - const std::string& name = p.first; - const auto& config = p.second; - parts.push_back(absl::StrCat(" ", name, "=", config.ToString(), "\n")); - } - parts.push_back(" }\n"); - parts.push_back("]\n"); + parts.push_back(vhost.ToString()); } parts.push_back("cluster_specifier_plugins={\n"); for (const auto& it : cluster_specifier_plugin_map) { diff --git a/src/core/ext/xds/xds_route_config.h b/src/core/ext/xds/xds_route_config.h index 96aeb83c8fb..e9cc39efc57 100644 --- a/src/core/ext/xds/xds_route_config.h +++ b/src/core/ext/xds/xds_route_config.h @@ -209,6 +209,7 @@ struct XdsRouteConfigResource : public XdsResourceType::ResourceData { return domains == other.domains && routes == other.routes && typed_per_filter_config == other.typed_per_filter_config; } + std::string ToString() const; }; std::vector virtual_hosts; diff --git a/src/core/ext/xds/xds_server_config_fetcher.cc b/src/core/ext/xds/xds_server_config_fetcher.cc index 1c6373f4dfa..6c911696013 100644 --- a/src/core/ext/xds/xds_server_config_fetcher.cc +++ b/src/core/ext/xds/xds_server_config_fetcher.cc @@ -231,15 +231,6 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager } private: - struct CertificateProviders { - // We need to save our own refs to the root and instance certificate - // providers since the xds certificate provider just stores a ref to their - // distributors. - RefCountedPtr root; - RefCountedPtr instance; - RefCountedPtr xds; - }; - class RouteConfigWatcher; struct RdsUpdateState { RouteConfigWatcher* watcher; @@ -277,7 +268,8 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager size_t rds_resources_yet_to_fetch_ ABSL_GUARDED_BY(mu_) = 0; std::map rds_map_ ABSL_GUARDED_BY(mu_); - std::map + std::map> certificate_providers_map_ ABSL_GUARDED_BY(mu_); }; @@ -809,10 +801,7 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: const XdsListenerResource::FilterChainData* filter_chain) { MutexLock lock(&mu_); auto it = certificate_providers_map_.find(filter_chain); - if (it != certificate_providers_map_.end()) { - return it->second.xds; - } - CertificateProviders certificate_providers; + if (it != certificate_providers_map_.end()) return it->second; // Configure root cert. absl::string_view root_provider_instance_name = filter_chain->downstream_tls_context.common_tls_context @@ -822,11 +811,12 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: filter_chain->downstream_tls_context.common_tls_context .certificate_validation_context.ca_certificate_provider_instance .certificate_name; + RefCountedPtr root_cert_provider; if (!root_provider_instance_name.empty()) { - certificate_providers.root = + root_cert_provider = xds_client_->certificate_provider_store() .CreateOrGetCertificateProvider(root_provider_instance_name); - if (certificate_providers.root == nullptr) { + if (root_cert_provider == nullptr) { return absl::NotFoundError( absl::StrCat("Certificate provider instance name: \"", root_provider_instance_name, "\" not recognized.")); @@ -839,33 +829,23 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: absl::string_view identity_provider_cert_name = filter_chain->downstream_tls_context.common_tls_context .tls_certificate_provider_instance.certificate_name; + RefCountedPtr identity_cert_provider; if (!identity_provider_instance_name.empty()) { - certificate_providers.instance = + identity_cert_provider = xds_client_->certificate_provider_store() .CreateOrGetCertificateProvider(identity_provider_instance_name); - if (certificate_providers.instance == nullptr) { + if (identity_cert_provider == nullptr) { return absl::NotFoundError( absl::StrCat("Certificate provider instance name: \"", identity_provider_instance_name, "\" not recognized.")); } } - certificate_providers.xds = MakeRefCounted(); - certificate_providers.xds->UpdateRootCertNameAndDistributor( - "", root_provider_cert_name, - certificate_providers.root == nullptr - ? nullptr - : certificate_providers.root->distributor()); - certificate_providers.xds->UpdateIdentityCertNameAndDistributor( - "", identity_provider_cert_name, - certificate_providers.instance == nullptr - ? nullptr - : certificate_providers.instance->distributor()); - certificate_providers.xds->UpdateRequireClientCertificate( - "", filter_chain->downstream_tls_context.require_client_certificate); - auto xds_certificate_provider = certificate_providers.xds; - certificate_providers_map_.emplace(filter_chain, - std::move(certificate_providers)); - return xds_certificate_provider; + auto xds_cert_provider = MakeRefCounted( + std::move(root_cert_provider), root_provider_cert_name, + std::move(identity_cert_provider), identity_provider_cert_name, + filter_chain->downstream_tls_context.require_client_certificate); + certificate_providers_map_.emplace(filter_chain, xds_cert_provider); + return xds_cert_provider; } void XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: diff --git a/src/core/lib/security/credentials/xds/xds_credentials.cc b/src/core/lib/security/credentials/xds/xds_credentials.cc index 957e36274b2..e93df47cc90 100644 --- a/src/core/lib/security/credentials/xds/xds_credentials.cc +++ b/src/core/lib/security/credentials/xds/xds_credentials.cc @@ -74,10 +74,8 @@ bool XdsVerifySubjectAlternativeNames( // XdsCertificateVerifier::XdsCertificateVerifier( - RefCountedPtr xds_certificate_provider, - std::string cluster_name) - : xds_certificate_provider_(std::move(xds_certificate_provider)), - cluster_name_(std::move(cluster_name)) {} + RefCountedPtr xds_certificate_provider) + : xds_certificate_provider_(std::move(xds_certificate_provider)) {} bool XdsCertificateVerifier::Verify( grpc_tls_custom_verification_check_request* request, @@ -86,15 +84,15 @@ bool XdsCertificateVerifier::Verify( if (!XdsVerifySubjectAlternativeNames( request->peer_info.san_names.uri_names, request->peer_info.san_names.uri_names_size, - xds_certificate_provider_->GetSanMatchers(cluster_name_)) && + xds_certificate_provider_->san_matchers()) && !XdsVerifySubjectAlternativeNames( request->peer_info.san_names.ip_names, request->peer_info.san_names.ip_names_size, - xds_certificate_provider_->GetSanMatchers(cluster_name_)) && + xds_certificate_provider_->san_matchers()) && !XdsVerifySubjectAlternativeNames( request->peer_info.san_names.dns_names, request->peer_info.san_names.dns_names_size, - xds_certificate_provider_->GetSanMatchers(cluster_name_))) { + xds_certificate_provider_->san_matchers())) { *sync_status = absl::Status( absl::StatusCode::kUnauthenticated, "SANs from certificate did not match SANs from xDS control plane"); @@ -108,9 +106,12 @@ void XdsCertificateVerifier::Cancel( int XdsCertificateVerifier::CompareImpl( const grpc_tls_certificate_verifier* other) const { auto* o = static_cast(other); - int r = QsortCompare(xds_certificate_provider_, o->xds_certificate_provider_); - if (r != 0) return r; - return cluster_name_.compare(o->cluster_name_); + if (xds_certificate_provider_ == nullptr || + o->xds_certificate_provider_ == nullptr) { + return QsortCompare(xds_certificate_provider_, + o->xds_certificate_provider_); + } + return xds_certificate_provider_->Compare(o->xds_certificate_provider_.get()); } UniqueTypeName XdsCertificateVerifier::type() const { @@ -140,12 +141,9 @@ XdsCredentials::create_security_connector( RefCountedPtr security_connector; auto xds_certificate_provider = args->GetObjectRef(); if (xds_certificate_provider != nullptr) { - std::string cluster_name( - args->GetString(GRPC_ARG_XDS_CLUSTER_NAME).value()); - const bool watch_root = - xds_certificate_provider->ProvidesRootCerts(cluster_name); + const bool watch_root = xds_certificate_provider->ProvidesRootCerts(); const bool watch_identity = - xds_certificate_provider->ProvidesIdentityCerts(cluster_name); + xds_certificate_provider->ProvidesIdentityCerts(); if (watch_root || watch_identity) { auto tls_credentials_options = MakeRefCounted(); @@ -153,16 +151,14 @@ XdsCredentials::create_security_connector( xds_certificate_provider); if (watch_root) { tls_credentials_options->set_watch_root_cert(true); - tls_credentials_options->set_root_cert_name(cluster_name); } if (watch_identity) { tls_credentials_options->set_watch_identity_pair(true); - tls_credentials_options->set_identity_cert_name(cluster_name); } tls_credentials_options->set_verify_server_cert(true); tls_credentials_options->set_certificate_verifier( - MakeRefCounted(xds_certificate_provider, - std::move(cluster_name))); + MakeRefCounted( + std::move(xds_certificate_provider))); tls_credentials_options->set_check_call_host(false); auto tls_credentials = MakeRefCounted(std::move(tls_credentials_options)); @@ -189,20 +185,17 @@ XdsServerCredentials::create_security_connector(const ChannelArgs& args) { auto xds_certificate_provider = args.GetObjectRef(); // Identity certs are a must for TLS. if (xds_certificate_provider != nullptr && - xds_certificate_provider->ProvidesIdentityCerts("")) { + xds_certificate_provider->ProvidesIdentityCerts()) { auto tls_credentials_options = MakeRefCounted(); tls_credentials_options->set_watch_identity_pair(true); tls_credentials_options->set_certificate_provider(xds_certificate_provider); - if (xds_certificate_provider->ProvidesRootCerts("")) { + if (xds_certificate_provider->ProvidesRootCerts()) { tls_credentials_options->set_watch_root_cert(true); - if (xds_certificate_provider->GetRequireClientCertificate("")) { - tls_credentials_options->set_cert_request_type( - GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY); - } else { - tls_credentials_options->set_cert_request_type( - GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); - } + tls_credentials_options->set_cert_request_type( + xds_certificate_provider->require_client_certificate() + ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY + : GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY); } else { // Do not request client certificate if there is no way to verify. tls_credentials_options->set_cert_request_type( diff --git a/src/core/lib/security/credentials/xds/xds_credentials.h b/src/core/lib/security/credentials/xds/xds_credentials.h index 7284d22fc7b..7b8ce1531a4 100644 --- a/src/core/lib/security/credentials/xds/xds_credentials.h +++ b/src/core/lib/security/credentials/xds/xds_credentials.h @@ -46,9 +46,8 @@ namespace grpc_core { class XdsCertificateVerifier : public grpc_tls_certificate_verifier { public: - XdsCertificateVerifier( - RefCountedPtr xds_certificate_provider, - std::string cluster_name); + explicit XdsCertificateVerifier( + RefCountedPtr xds_certificate_provider); bool Verify(grpc_tls_custom_verification_check_request* request, std::function, @@ -61,7 +60,6 @@ class XdsCertificateVerifier : public grpc_tls_certificate_verifier { int CompareImpl(const grpc_tls_certificate_verifier* other) const override; RefCountedPtr xds_certificate_provider_; - std::string cluster_name_; }; class XdsCredentials final : public grpc_channel_credentials { diff --git a/src/core/plugin_registry/grpc_plugin_registry_extra.cc b/src/core/plugin_registry/grpc_plugin_registry_extra.cc index 43a98d8bb35..bf7fa8abb50 100644 --- a/src/core/plugin_registry/grpc_plugin_registry_extra.cc +++ b/src/core/plugin_registry/grpc_plugin_registry_extra.cc @@ -34,8 +34,6 @@ extern void RegisterXdsClusterManagerLbPolicy( CoreConfiguration::Builder* builder); extern void RegisterXdsClusterImplLbPolicy(CoreConfiguration::Builder* builder); extern void RegisterCdsLbPolicy(CoreConfiguration::Builder* builder); -extern void RegisterXdsClusterResolverLbPolicy( - CoreConfiguration::Builder* builder); extern void RegisterXdsOverrideHostLbPolicy( CoreConfiguration::Builder* builder); extern void RegisterXdsWrrLocalityLbPolicy(CoreConfiguration::Builder* builder); @@ -58,7 +56,6 @@ void RegisterExtraFilters(CoreConfiguration::Builder* builder) { RegisterXdsClusterManagerLbPolicy(builder); RegisterXdsClusterImplLbPolicy(builder); RegisterCdsLbPolicy(builder); - RegisterXdsClusterResolverLbPolicy(builder); RegisterXdsOverrideHostLbPolicy(builder); RegisterXdsWrrLocalityLbPolicy(builder); RegisterRingHashLbPolicy(builder); diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index bd30eef8d32..971a75742d4 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -54,7 +54,6 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', @@ -73,7 +72,9 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc', 'src/core/ext/filters/client_channel/resolver/polling_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc', 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc', + 'src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc', 'src/core/ext/filters/client_channel/retry_filter.cc', 'src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc', 'src/core/ext/filters/client_channel/retry_service_config.cc', diff --git a/test/core/client_channel/client_channel_service_config_test.cc b/test/core/client_channel/client_channel_service_config_test.cc index f709c8199a2..e265653daea 100644 --- a/test/core/client_channel/client_channel_service_config_test.cc +++ b/test/core/client_channel/client_channel_service_config_test.cc @@ -83,29 +83,6 @@ TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigGrpclb) { EXPECT_EQ(lb_config->name(), "grpclb"); } -TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigXds) { - const char* test_json = - "{\n" - " \"loadBalancingConfig\":[\n" - " { \"does_not_exist\":{} },\n" - " { \"xds_cluster_resolver_experimental\":{\n" - " \"discoveryMechanisms\": [\n" - " { \"clusterName\": \"foo\",\n" - " \"type\": \"EDS\"\n" - " } ],\n" - " \"xdsLbPolicy\": [{\"round_robin\":{}}]\n" - " } }\n" - " ]\n" - "}"; - auto service_config = ServiceConfigImpl::Create(ChannelArgs(), test_json); - ASSERT_TRUE(service_config.ok()) << service_config.status(); - const auto* parsed_config = - static_cast( - (*service_config)->GetGlobalParsedConfig(parser_index_)); - auto lb_config = parsed_config->parsed_lb_config(); - EXPECT_EQ(lb_config->name(), "xds_cluster_resolver_experimental"); -} - TEST_F(ClientChannelParserTest, UnknownLoadBalancingConfig) { const char* test_json = "{\"loadBalancingConfig\": [{\"unknown\":{}}]}"; auto service_config = ServiceConfigImpl::Create(ChannelArgs(), test_json); @@ -163,15 +140,15 @@ TEST_F(ClientChannelParserTest, UnknownLoadBalancingPolicy) { << service_config.status(); } -TEST_F(ClientChannelParserTest, LoadBalancingPolicyXdsNotAllowed) { - const char* test_json = - "{\"loadBalancingPolicy\":\"xds_cluster_resolver_experimental\"}"; +TEST_F(ClientChannelParserTest, + LegacyLoadBalancingPolicySpecifiesPolicyThatRequiresConfig) { + const char* test_json = "{\"loadBalancingPolicy\":\"rls_experimental\"}"; auto service_config = ServiceConfigImpl::Create(ChannelArgs(), test_json); EXPECT_EQ(service_config.status().code(), absl::StatusCode::kInvalidArgument); EXPECT_EQ(service_config.status().message(), "errors validating service config: [" "field:loadBalancingPolicy error:LB policy " - "\"xds_cluster_resolver_experimental\" requires a config. Please " + "\"rls_experimental\" requires a config. Please " "use loadBalancingConfig instead.]") << service_config.status(); } diff --git a/test/core/client_channel/lb_policy/lb_policy_test_lib.h b/test/core/client_channel/lb_policy/lb_policy_test_lib.h index 27d7f38285b..ff254a4596d 100644 --- a/test/core/client_channel/lb_policy/lb_policy_test_lib.h +++ b/test/core/client_channel/lb_policy/lb_policy_test_lib.h @@ -542,7 +542,8 @@ class LoadBalancingPolicyTest : public ::testing::Test { const ChannelArgs& /*per_address_args*/, const ChannelArgs& args) override { // TODO(roth): Need to use per_address_args here. - SubchannelKey key(address, args); + SubchannelKey key( + address, args.RemoveAllKeysWithPrefix(GRPC_ARG_NO_SUBCHANNEL_PREFIX)); auto it = test_->subchannel_pool_.find(key); if (it == test_->subchannel_pool_.end()) { auto address_uri = grpc_sockaddr_to_uri(&address); @@ -771,11 +772,13 @@ class LoadBalancingPolicyTest : public ::testing::Test { // Constructs an update containing a list of endpoints. LoadBalancingPolicy::UpdateArgs BuildUpdate( absl::Span endpoints, - RefCountedPtr config) { + RefCountedPtr config, + ChannelArgs args = ChannelArgs()) { LoadBalancingPolicy::UpdateArgs update; update.addresses = std::make_shared( EndpointAddressesList(endpoints.begin(), endpoints.end())); update.config = std::move(config); + update.args = std::move(args); return update; } @@ -791,9 +794,10 @@ class LoadBalancingPolicyTest : public ::testing::Test { // Convenient overload that takes a flat address list. LoadBalancingPolicy::UpdateArgs BuildUpdate( absl::Span addresses, - RefCountedPtr config) { + RefCountedPtr config, + ChannelArgs args = ChannelArgs()) { return BuildUpdate(MakeEndpointAddressesListFromAddressList(addresses), - std::move(config)); + std::move(config), std::move(args)); } // Applies the update on the LB policy. @@ -1161,7 +1165,8 @@ class LoadBalancingPolicyTest : public ::testing::Test { // Expect startup with RR with a set of addresses. RefCountedPtr ExpectRoundRobinStartup( - absl::Span endpoints) { + absl::Span endpoints, + SourceLocation location = SourceLocation()) { GPR_ASSERT(!endpoints.empty()); // There should be a subchannel for every address. // We will wind up connecting to the first address for every endpoint. @@ -1178,7 +1183,9 @@ class LoadBalancingPolicyTest : public ::testing::Test { const grpc_resolved_address& address = endpoint.addresses()[i]; std::string address_str = grpc_sockaddr_to_uri(&address).value(); auto* subchannel = FindSubchannel(address_str); - EXPECT_NE(subchannel, nullptr); + EXPECT_NE(subchannel, nullptr) + << address_str << "\n" + << location.file() << ":" << location.line(); if (subchannel == nullptr) return nullptr; endpoint_subchannels.back().push_back(subchannel); if (i == 0) { @@ -1190,16 +1197,19 @@ class LoadBalancingPolicyTest : public ::testing::Test { // We should request a connection to the first address of each endpoint, // and not to any of the subsequent addresses. for (const auto& subchannels : endpoint_subchannels) { - EXPECT_TRUE(subchannels[0]->ConnectionRequested()); + EXPECT_TRUE(subchannels[0]->ConnectionRequested()) + << location.file() << ":" << location.line(); for (size_t i = 1; i < subchannels.size(); ++i) { - EXPECT_FALSE(subchannels[i]->ConnectionRequested()); + EXPECT_FALSE(subchannels[i]->ConnectionRequested()) + << "i=" << i << "\n" + << location.file() << ":" << location.line(); } } // The subchannels that we've asked to connect should report // CONNECTING state. for (size_t i = 0; i < endpoint_subchannels.size(); ++i) { endpoint_subchannels[i][0]->SetConnectivityState(GRPC_CHANNEL_CONNECTING); - if (i == 0) ExpectConnectingUpdate(); + if (i == 0) ExpectConnectingUpdate(location); } // The connection attempts should succeed. RefCountedPtr picker; @@ -1209,8 +1219,9 @@ class LoadBalancingPolicyTest : public ::testing::Test { // When the first subchannel becomes READY, accept any number of // CONNECTING updates with a picker that queues followed by a READY // update with a picker that repeatedly returns only the first address. - picker = WaitForConnected(); - ExpectRoundRobinPicks(picker.get(), {chosen_addresses[0]}); + picker = WaitForConnected(location); + ExpectRoundRobinPicks(picker.get(), {chosen_addresses[0]}, {}, 3, + location); } else { // When each subsequent subchannel becomes READY, we accept any number // of READY updates where the picker returns only the previously @@ -1219,7 +1230,8 @@ class LoadBalancingPolicyTest : public ::testing::Test { // connected subchannel. picker = WaitForRoundRobinListChange( absl::MakeSpan(chosen_addresses).subspan(0, i), - absl::MakeSpan(chosen_addresses).subspan(0, i + 1)); + absl::MakeSpan(chosen_addresses).subspan(0, i + 1), {}, 3, + location); } } return picker; @@ -1228,9 +1240,10 @@ class LoadBalancingPolicyTest : public ::testing::Test { // A convenient override that takes a flat list of addresses, one per // endpoint. RefCountedPtr ExpectRoundRobinStartup( - absl::Span addresses) { + absl::Span addresses, + SourceLocation location = SourceLocation()) { return ExpectRoundRobinStartup( - MakeEndpointAddressesListFromAddressList(addresses)); + MakeEndpointAddressesListFromAddressList(addresses), location); } // Expects zero or more picker updates, each of which returns diff --git a/test/core/client_channel/lb_policy/xds_override_host_lb_config_parser_test.cc b/test/core/client_channel/lb_policy/xds_override_host_lb_config_parser_test.cc index a03987fc993..fcf77ef8674 100644 --- a/test/core/client_channel/lb_policy/xds_override_host_lb_config_parser_test.cc +++ b/test/core/client_channel/lb_policy/xds_override_host_lb_config_parser_test.cc @@ -43,12 +43,10 @@ TEST(XdsOverrideHostConfigParsingTest, ValidConfig) { "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" + " \"clusterName\": \"foo\",\n" " \"childPolicy\":[\n" " {\"grpclb\":{}}\n" - " ],\n" - " \"overrideHostStatus\": [\n" - " \"DRAINING\", \"HEALTHY\", \"UNKNOWN\"" - " ]" + " ]\n" " }\n" " }]\n" "}\n"; @@ -64,16 +62,11 @@ TEST(XdsOverrideHostConfigParsingTest, ValidConfig) { auto lb_config = global_config->parsed_lb_config(); ASSERT_NE(lb_config, nullptr); ASSERT_EQ(lb_config->name(), XdsOverrideHostLbConfig::Name()); - auto override_host_lb_config = - lb_config.TakeAsSubclass(); - EXPECT_EQ(override_host_lb_config->override_host_status_set(), - XdsHealthStatusSet({ - XdsHealthStatus(XdsHealthStatus::HealthStatus::kDraining), - XdsHealthStatus(XdsHealthStatus::HealthStatus::kHealthy), - XdsHealthStatus(XdsHealthStatus::HealthStatus::kUnknown), - })); + auto* override_host_lb_config = + static_cast(lb_config.get()); + EXPECT_EQ(override_host_lb_config->cluster_name(), "foo"); ASSERT_NE(override_host_lb_config->child_config(), nullptr); - ASSERT_EQ(override_host_lb_config->child_config()->name(), "grpclb"); + EXPECT_EQ(override_host_lb_config->child_config()->name(), "grpclb"); } TEST(XdsOverrideHostConfigParsingTest, ValidConfigWithRR) { @@ -81,6 +74,7 @@ TEST(XdsOverrideHostConfigParsingTest, ValidConfigWithRR) { "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" + " \"clusterName\": \"foo\",\n" " \"childPolicy\":[\n" " {\"round_robin\":{}}\n" " ]\n" @@ -99,77 +93,32 @@ TEST(XdsOverrideHostConfigParsingTest, ValidConfigWithRR) { auto lb_config = global_config->parsed_lb_config(); ASSERT_NE(lb_config, nullptr); ASSERT_EQ(lb_config->name(), XdsOverrideHostLbConfig::Name()); - auto override_host_lb_config = - lb_config.TakeAsSubclass(); - ASSERT_NE(override_host_lb_config->child_config(), nullptr); - ASSERT_EQ(override_host_lb_config->child_config()->name(), "round_robin"); -} - -TEST(XdsOverrideHostConfigParsingTest, ValidConfigNoDraining) { - const char* service_config_json = - "{\n" - " \"loadBalancingConfig\":[{\n" - " \"xds_override_host_experimental\":{\n" - " \"childPolicy\":[\n" - " {\"grpclb\":{}}\n" - " ],\n" - " \"overrideHostStatus\": [\n" - " \"HEALTHY\", \"UNKNOWN\"" - " ]" - " }\n" - " }]\n" - "}\n"; - auto service_config = - ServiceConfigImpl::Create(ChannelArgs(), service_config_json); - ASSERT_TRUE(service_config.ok()); - EXPECT_NE(*service_config, nullptr); - auto global_config = static_cast( - (*service_config) - ->GetGlobalParsedConfig( - ClientChannelServiceConfigParser::ParserIndex())); - ASSERT_NE(global_config, nullptr); - auto lb_config = global_config->parsed_lb_config(); - ASSERT_NE(lb_config, nullptr); - ASSERT_EQ(lb_config->name(), XdsOverrideHostLbConfig::Name()); - auto override_host_lb_config = - lb_config.TakeAsSubclass(); - EXPECT_EQ(override_host_lb_config->override_host_status_set(), - XdsHealthStatusSet( - {XdsHealthStatus(XdsHealthStatus::HealthStatus::kHealthy), - XdsHealthStatus(XdsHealthStatus::HealthStatus::kUnknown)})); + auto* override_host_lb_config = + static_cast(lb_config.get()); + EXPECT_EQ(override_host_lb_config->cluster_name(), "foo"); ASSERT_NE(override_host_lb_config->child_config(), nullptr); - ASSERT_EQ(override_host_lb_config->child_config()->name(), "grpclb"); + EXPECT_EQ(override_host_lb_config->child_config()->name(), "round_robin"); } -TEST(XdsOverrideHostConfigParsingTest, ValidConfigNoOverrideHostStatuses) { +TEST(XdsOverrideHostConfigParsingTest, MissingClusterName) { const char* service_config_json = "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" " \"childPolicy\":[\n" - " {\"grpclb\":{}}\n" - " ]" + " {\"round_robin\":{}}\n" + " ]\n" " }\n" " }]\n" "}\n"; auto service_config = ServiceConfigImpl::Create(ChannelArgs(), service_config_json); - ASSERT_TRUE(service_config.ok()); - EXPECT_NE(*service_config, nullptr); - auto global_config = static_cast( - (*service_config)->GetGlobalParsedConfig(0)); - ASSERT_NE(global_config, nullptr); - auto lb_config = global_config->parsed_lb_config(); - ASSERT_NE(lb_config, nullptr); - ASSERT_EQ(lb_config->name(), XdsOverrideHostLbConfig::Name()); - auto override_host_lb_config = - lb_config.TakeAsSubclass(); - EXPECT_EQ(override_host_lb_config->override_host_status_set(), - XdsHealthStatusSet( - {XdsHealthStatus(XdsHealthStatus::HealthStatus::kHealthy), - XdsHealthStatus(XdsHealthStatus::HealthStatus::kUnknown)})); - ASSERT_NE(override_host_lb_config->child_config(), nullptr); - EXPECT_EQ(override_host_lb_config->child_config()->name(), "grpclb"); + ASSERT_FALSE(service_config.ok()); + EXPECT_EQ(service_config.status(), + absl::InvalidArgumentError( + "errors validating service config: [field:loadBalancingConfig " + "error:errors validating xds_override_host LB policy config: " + "[field:clusterName error:field not present]]")); } TEST(XdsOverrideHostConfigParsingTest, ReportsMissingChildPolicyField) { @@ -177,6 +126,7 @@ TEST(XdsOverrideHostConfigParsingTest, ReportsMissingChildPolicyField) { "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" + " \"clusterName\": \"foo\"\n" " }\n" " }]\n" "}\n"; @@ -195,6 +145,7 @@ TEST(XdsOverrideHostConfigParsingTest, ReportsChildPolicyShouldBeArray) { "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" + " \"clusterName\": \"foo\",\n" " \"childPolicy\":{\n" " \"grpclb\":{}\n" " }\n" @@ -216,6 +167,7 @@ TEST(XdsOverrideHostConfigParsingTest, ReportsEmptyChildPolicyArray) { "{\n" " \"loadBalancingConfig\":[{\n" " \"xds_override_host_experimental\":{\n" + " \"clusterName\": \"foo\",\n" " \"childPolicy\":[\n" " ]\n" " }\n" @@ -231,29 +183,6 @@ TEST(XdsOverrideHostConfigParsingTest, ReportsEmptyChildPolicyArray) { "[field:childPolicy error:No known policies in list: ]]")); } -TEST(XdsOverrideHostConfigParsingTest, UnrecognizedHostStatus) { - const char* service_config_json = - "{\n" - " \"loadBalancingConfig\":[{\n" - " \"xds_override_host_experimental\":{\n" - " \"childPolicy\":[\n" - " {\"grpclb\":{}}\n" - " ],\n" - " \"overrideHostStatus\": [\n" - " \"NOTASTATUS\"" - " ]" - " }\n" - " }]\n" - "}\n"; - auto service_config = - ServiceConfigImpl::Create(ChannelArgs(), service_config_json); - ASSERT_FALSE(service_config.ok()) << service_config.status(); - EXPECT_EQ(service_config.status(), - absl::InvalidArgumentError( - "errors validating service config: [field:loadBalancingConfig " - "error:errors validating xds_override_host LB policy config: " - "[field:overrideHostStatus[0] error:invalid host status]]")); -} } // namespace } // namespace testing } // namespace grpc_core diff --git a/test/core/client_channel/lb_policy/xds_override_host_test.cc b/test/core/client_channel/lb_policy/xds_override_host_test.cc index 7e9d45d7f63..36999dc7b95 100644 --- a/test/core/client_channel/lb_policy/xds_override_host_test.cc +++ b/test/core/client_channel/lb_policy/xds_override_host_test.cc @@ -37,6 +37,7 @@ #include #include +#include "src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h" #include "src/core/ext/filters/stateful_session/stateful_session_filter.h" #include "src/core/ext/xds/xds_health_status.h" #include "src/core/lib/channel/channel_args.h" @@ -57,31 +58,59 @@ class XdsOverrideHostTest : public LoadBalancingPolicyTest { XdsOverrideHostTest() : LoadBalancingPolicyTest("xds_override_host_experimental") {} - static RefCountedPtr MakeXdsOverrideHostConfig( - absl::Span override_host_status = {"UNKNOWN", - "HEALTHY"}, - std::string child_policy = "round_robin") { - Json child_policy_config = - Json::FromObject({{child_policy, Json::FromObject({})}}); - Json::Array override_host_status_array; - for (const absl::string_view host_status : override_host_status) { - override_host_status_array.push_back( - Json::FromString(std::string(host_status))); + static RefCountedPtr MakeXdsConfig( + absl::Span override_host_statuses = {"UNKNOWN", + "HEALTHY"}, + std::string cluster_name = "cluster_name") { + auto cluster_resource = std::make_shared(); + for (const absl::string_view host_status : override_host_statuses) { + cluster_resource->override_host_statuses.Add( + XdsHealthStatus::FromString(host_status).value()); } - return MakeConfig(Json::FromArray({Json::FromObject( + auto xds_config = MakeRefCounted(); + xds_config->clusters[cluster_name].emplace( + cluster_name, std::move(cluster_resource), nullptr, ""); + return xds_config; + } + + absl::Status UpdateXdsOverrideHostPolicy( + absl::Span endpoints, + absl::Span override_host_statuses = {"UNKNOWN", + "HEALTHY"}, + std::string cluster_name = "cluster_name", + std::string child_policy = "round_robin") { + auto config = MakeConfig(Json::FromArray({Json::FromObject( {{"xds_override_host_experimental", Json::FromObject( - {{"childPolicy", Json::FromArray({child_policy_config})}, - {"overrideHostStatus", - Json::FromArray(override_host_status_array)}})}})})); + {{"clusterName", Json::FromString(cluster_name)}, + {"childPolicy", + Json::FromArray({Json::FromObject( + {{child_policy, Json::FromObject({})}})})}})}})})); + auto xds_config = MakeXdsConfig(override_host_statuses, cluster_name); + return ApplyUpdate( + BuildUpdate(endpoints, std::move(config), + ChannelArgs().SetObject(std::move(xds_config))), + lb_policy()); + } + + absl::Status UpdateXdsOverrideHostPolicy( + absl::Span addresses, + absl::Span override_host_statuses = {"UNKNOWN", + "HEALTHY"}, + std::string cluster_name = "cluster_name", + std::string child_policy = "round_robin") { + return UpdateXdsOverrideHostPolicy( + MakeEndpointAddressesListFromAddressList(addresses), + override_host_statuses, std::move(cluster_name), + std::move(child_policy)); } RefCountedPtr - ExpectStartupWithRoundRobin(absl::Span addresses) { - EXPECT_EQ(ApplyUpdate(BuildUpdate(addresses, MakeXdsOverrideHostConfig()), - lb_policy()), - absl::OkStatus()); - return ExpectRoundRobinStartup(addresses); + ExpectStartupWithRoundRobin(absl::Span addresses, + SourceLocation location = SourceLocation()) { + EXPECT_EQ(UpdateXdsOverrideHostPolicy(addresses), absl::OkStatus()) + << location.file() << ":" << location.line(); + return ExpectRoundRobinStartup(addresses, location); } EndpointAddresses MakeAddressWithHealthStatus( @@ -97,16 +126,13 @@ class XdsOverrideHostTest : public LoadBalancingPolicyTest { addresses_and_statuses, absl::Span override_host_status = {"UNKNOWN", "HEALTHY"}) { - LoadBalancingPolicy::UpdateArgs update; - update.config = MakeXdsOverrideHostConfig(override_host_status); EndpointAddressesList endpoints; for (auto address_and_status : addresses_and_statuses) { endpoints.push_back(MakeAddressWithHealthStatus( address_and_status.first, address_and_status.second)); } - update.addresses = - std::make_shared(std::move(endpoints)); - EXPECT_EQ(ApplyUpdate(update, lb_policy()), absl::OkStatus()); + EXPECT_EQ(UpdateXdsOverrideHostPolicy(endpoints, override_host_status), + absl::OkStatus()); } struct OverrideHostAttributeStorage { @@ -233,9 +259,7 @@ TEST_F(XdsOverrideHostTest, SubchannelsComeAndGo) { auto* address1_attribute = MakeOverrideHostAttribute(kAddresses[1]); ExpectOverridePicks(picker.get(), address1_attribute, kAddresses[1]); // The override address is removed. - EXPECT_EQ(ApplyUpdate(BuildUpdate({kAddresses[0], kAddresses[2]}, - MakeXdsOverrideHostConfig()), - lb_policy()), + EXPECT_EQ(UpdateXdsOverrideHostPolicy({kAddresses[0], kAddresses[2]}), absl::OkStatus()); picker = WaitForRoundRobinListChange(kAddresses, {kAddresses[0], kAddresses[2]}); @@ -244,9 +268,7 @@ TEST_F(XdsOverrideHostTest, SubchannelsComeAndGo) { ExpectRoundRobinPicksWithAttribute(picker.get(), address1_attribute, {kAddresses[0], kAddresses[2]}); // The override address comes back. - EXPECT_EQ(ApplyUpdate(BuildUpdate({kAddresses[1], kAddresses[2]}, - MakeXdsOverrideHostConfig()), - lb_policy()), + EXPECT_EQ(UpdateXdsOverrideHostPolicy({kAddresses[1], kAddresses[2]}), absl::OkStatus()); picker = WaitForRoundRobinListChange({kAddresses[0], kAddresses[2]}, {kAddresses[1], kAddresses[2]}); @@ -475,9 +497,7 @@ TEST_F(XdsOverrideHostTest, MultipleAddressesPerEndpoint) { MakeEndpointAddresses(kEndpoint1Addresses), MakeEndpointAddresses(kEndpoint2Addresses), MakeEndpointAddresses(kEndpoint3Addresses)}; - EXPECT_EQ(ApplyUpdate(BuildUpdate(kEndpoints, MakeXdsOverrideHostConfig()), - lb_policy()), - absl::OkStatus()); + EXPECT_EQ(UpdateXdsOverrideHostPolicy(kEndpoints), absl::OkStatus()); auto picker = ExpectRoundRobinStartup(kEndpoints); ASSERT_NE(picker, nullptr); // Check that the host is overridden. diff --git a/test/core/security/grpc_tls_credentials_options_comparator_test.cc b/test/core/security/grpc_tls_credentials_options_comparator_test.cc index 3aeff611c86..6971f0c20e5 100644 --- a/test/core/security/grpc_tls_credentials_options_comparator_test.cc +++ b/test/core/security/grpc_tls_credentials_options_comparator_test.cc @@ -75,7 +75,7 @@ TEST(TlsCredentialsOptionsComparatorTest, DifferentCertificateVerifier) { auto* options_1 = grpc_tls_credentials_options_create(); auto* options_2 = grpc_tls_credentials_options_create(); options_1->set_certificate_verifier(MakeRefCounted()); - options_2->set_certificate_verifier(MakeRefCounted(nullptr, "")); + options_2->set_certificate_verifier(MakeRefCounted(nullptr)); EXPECT_FALSE(*options_1 == *options_2); EXPECT_FALSE(*options_2 == *options_1); delete options_1; diff --git a/test/core/security/xds_credentials_test.cc b/test/core/security/xds_credentials_test.cc index 280d6ac07d1..7f54abe7002 100644 --- a/test/core/security/xds_credentials_test.cc +++ b/test/core/security/xds_credentials_test.cc @@ -290,24 +290,17 @@ TEST(XdsSanMatchingTest, RegexMatch) { } TEST(XdsCertificateVerifierTest, CompareSuccess) { - XdsCertificateVerifier verifier_1(nullptr, ""); - XdsCertificateVerifier verifier_2(nullptr, ""); + XdsCertificateVerifier verifier_1(nullptr); + XdsCertificateVerifier verifier_2(nullptr); EXPECT_EQ(verifier_1.Compare(&verifier_2), 0); EXPECT_EQ(verifier_2.Compare(&verifier_1), 0); } TEST(XdsCertificateVerifierTest, CompareFailureDifferentCertificateProviders) { - XdsCertificateVerifier verifier_1(MakeRefCounted(), - ""); - XdsCertificateVerifier verifier_2(MakeRefCounted(), - ""); - EXPECT_NE(verifier_1.Compare(&verifier_2), 0); - EXPECT_NE(verifier_2.Compare(&verifier_1), 0); -} - -TEST(XdsCertificateVerifierTest, CompareFailureDifferentClusterNames) { - XdsCertificateVerifier verifier_1(nullptr, "cluster1"); - XdsCertificateVerifier verifier_2(nullptr, "cluster2"); + XdsCertificateVerifier verifier_1( + MakeRefCounted(nullptr, "", nullptr, "", false)); + XdsCertificateVerifier verifier_2( + MakeRefCounted(nullptr, "", nullptr, "", false)); EXPECT_NE(verifier_1.Compare(&verifier_2), 0); EXPECT_NE(verifier_2.Compare(&verifier_1), 0); } diff --git a/test/core/xds/xds_certificate_provider_test.cc b/test/core/xds/xds_certificate_provider_test.cc index f2090e6bd15..9e4d3f94713 100644 --- a/test/core/xds/xds_certificate_provider_test.cc +++ b/test/core/xds/xds_certificate_provider_test.cc @@ -25,6 +25,7 @@ #include +#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/status_helper.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/security/security_connector/ssl_utils.h" @@ -52,6 +53,28 @@ PemKeyCertPairList MakeKeyCertPairsType2() { return MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2); } +class TestCertProvider : public grpc_tls_certificate_provider { + public: + TestCertProvider() + : distributor_(MakeRefCounted()) {} + + UniqueTypeName type() const override { + static UniqueTypeName::Factory kFactory("Xds"); + return kFactory.Create(); + } + + RefCountedPtr distributor() const override { + return distributor_; + } + + private: + int CompareImpl(const grpc_tls_certificate_provider* other) const override { + return QsortCompare(this, static_cast(other)); + } + + RefCountedPtr distributor_; +}; + class TestCertificatesWatcher : public grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface { public: @@ -102,14 +125,10 @@ class TestCertificatesWatcher TEST( XdsCertificateProviderTest, RootCertDistributorDifferentFromIdentityCertDistributorDifferentCertNames) { - auto root_cert_distributor = - MakeRefCounted(); - auto identity_cert_distributor = - MakeRefCounted(); - XdsCertificateProvider provider; - provider.UpdateRootCertNameAndDistributor("", "root", root_cert_distributor); - provider.UpdateIdentityCertNameAndDistributor("", "identity", - identity_cert_distributor); + auto root_provider = MakeRefCounted(); + auto identity_provider = MakeRefCounted(); + XdsCertificateProvider provider(root_provider, "root", identity_provider, + "identity", {}); auto* watcher = new TestCertificatesWatcher; provider.distributor()->WatchTlsCertificates( std::unique_ptr(watcher), "", ""); @@ -118,15 +137,16 @@ TEST( EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Update both root certs and identity certs - root_cert_distributor->SetKeyMaterials("root", kRootCert1, absl::nullopt); - identity_cert_distributor->SetKeyMaterials("identity", absl::nullopt, - MakeKeyCertPairsType1()); + root_provider->distributor()->SetKeyMaterials("root", kRootCert1, + absl::nullopt); + identity_provider->distributor()->SetKeyMaterials("identity", absl::nullopt, + MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Second update for just root certs - root_cert_distributor->SetKeyMaterials( + root_provider->distributor()->SetKeyMaterials( "root", kRootCert2, MakeKeyCertPairsType2() /* does not have an effect */); EXPECT_EQ(watcher->root_certs(), kRootCert2); @@ -134,7 +154,7 @@ TEST( EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Second update for identity certs - identity_cert_distributor->SetKeyMaterials( + identity_provider->distributor()->SetKeyMaterials( "identity", kRootCert1 /* does not have an effect */, MakeKeyCertPairsType2()); EXPECT_EQ(watcher->root_certs(), kRootCert2); @@ -142,9 +162,9 @@ TEST( EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Set error for both root and identity - root_cert_distributor->SetErrorForCert( + root_provider->distributor()->SetErrorForCert( "root", GRPC_ERROR_CREATE(kRootErrorMessage), absl::nullopt); - identity_cert_distributor->SetErrorForCert( + identity_provider->distributor()->SetErrorForCert( "identity", absl::nullopt, GRPC_ERROR_CREATE(kIdentityErrorMessage)); EXPECT_EQ(watcher->root_certs(), kRootCert2); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); @@ -153,7 +173,8 @@ TEST( EXPECT_THAT(StatusToString(watcher->identity_cert_error()), ::testing::HasSubstr(kIdentityErrorMessage)); // Send an update for root certs. Test that the root cert error is reset. - root_cert_distributor->SetKeyMaterials("root", kRootCert1, absl::nullopt); + root_provider->distributor()->SetKeyMaterials("root", kRootCert1, + absl::nullopt); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); @@ -161,8 +182,8 @@ TEST( ::testing::HasSubstr(kIdentityErrorMessage)); // Send an update for identity certs. Test that the identity cert error is // reset. - identity_cert_distributor->SetKeyMaterials("identity", absl::nullopt, - MakeKeyCertPairsType1()); + identity_provider->distributor()->SetKeyMaterials("identity", absl::nullopt, + MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); @@ -171,14 +192,10 @@ TEST( TEST(XdsCertificateProviderTest, RootCertDistributorDifferentFromIdentityCertDistributorSameCertNames) { - auto root_cert_distributor = - MakeRefCounted(); - auto identity_cert_distributor = - MakeRefCounted(); - XdsCertificateProvider provider; - provider.UpdateRootCertNameAndDistributor("", "test", root_cert_distributor); - provider.UpdateIdentityCertNameAndDistributor("", "test", - identity_cert_distributor); + auto root_provider = MakeRefCounted(); + auto identity_provider = MakeRefCounted(); + XdsCertificateProvider provider(root_provider, "test", identity_provider, + "test", {}); auto* watcher = new TestCertificatesWatcher; provider.distributor()->WatchTlsCertificates( std::unique_ptr(watcher), "", ""); @@ -187,30 +204,32 @@ TEST(XdsCertificateProviderTest, EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Update both root certs and identity certs - root_cert_distributor->SetKeyMaterials("test", kRootCert1, absl::nullopt); - identity_cert_distributor->SetKeyMaterials("test", absl::nullopt, - MakeKeyCertPairsType1()); + root_provider->distributor()->SetKeyMaterials("test", kRootCert1, + absl::nullopt); + identity_provider->distributor()->SetKeyMaterials("test", absl::nullopt, + MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Second update for just root certs - root_cert_distributor->SetKeyMaterials("test", kRootCert2, absl::nullopt); + root_provider->distributor()->SetKeyMaterials("test", kRootCert2, + absl::nullopt); EXPECT_EQ(watcher->root_certs(), kRootCert2); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Second update for identity certs - identity_cert_distributor->SetKeyMaterials("test", absl::nullopt, - MakeKeyCertPairsType2()); + identity_provider->distributor()->SetKeyMaterials("test", absl::nullopt, + MakeKeyCertPairsType2()); EXPECT_EQ(watcher->root_certs(), kRootCert2); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Set error for both root and identity - root_cert_distributor->SetErrorForCert( + root_provider->distributor()->SetErrorForCert( "test", GRPC_ERROR_CREATE(kRootErrorMessage), absl::nullopt); - identity_cert_distributor->SetErrorForCert( + identity_provider->distributor()->SetErrorForCert( "test", absl::nullopt, GRPC_ERROR_CREATE(kIdentityErrorMessage)); EXPECT_EQ(watcher->root_certs(), kRootCert2); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); @@ -219,7 +238,8 @@ TEST(XdsCertificateProviderTest, EXPECT_THAT(StatusToString(watcher->identity_cert_error()), ::testing::HasSubstr(kIdentityErrorMessage)); // Send an update for root certs. Test that the root cert error is reset. - root_cert_distributor->SetKeyMaterials("test", kRootCert1, absl::nullopt); + root_provider->distributor()->SetKeyMaterials("test", kRootCert1, + absl::nullopt); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); @@ -227,25 +247,25 @@ TEST(XdsCertificateProviderTest, ::testing::HasSubstr(kIdentityErrorMessage)); // Send an update for identity certs. Test that the identity cert error is // reset. - identity_cert_distributor->SetKeyMaterials("test", absl::nullopt, - MakeKeyCertPairsType1()); + identity_provider->distributor()->SetKeyMaterials("test", absl::nullopt, + MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_certs(), kRootCert1); EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); // Test update on unwatched cert name - identity_cert_distributor->SetKeyMaterials("identity", kRootCert2, - MakeKeyCertPairsType2()); - root_cert_distributor->SetKeyMaterials("root", kRootCert1, - MakeKeyCertPairsType1()); + identity_provider->distributor()->SetKeyMaterials("identity", kRootCert2, + MakeKeyCertPairsType2()); + root_provider->distributor()->SetKeyMaterials("root", kRootCert1, + MakeKeyCertPairsType1()); } TEST(XdsCertificateProviderTest, RootCertDistributorSameAsIdentityCertDistributorDifferentCertNames) { - auto distributor = MakeRefCounted(); - XdsCertificateProvider provider; - provider.UpdateRootCertNameAndDistributor("", "root", distributor); - provider.UpdateIdentityCertNameAndDistributor("", "identity", distributor); + auto root_and_identity_provider = MakeRefCounted(); + auto distributor = root_and_identity_provider->distributor(); + XdsCertificateProvider provider(root_and_identity_provider, "root", + root_and_identity_provider, "identity", {}); auto* watcher = new TestCertificatesWatcher; provider.distributor()->WatchTlsCertificates( std::unique_ptr(watcher), "", ""); @@ -306,10 +326,10 @@ TEST(XdsCertificateProviderTest, TEST(XdsCertificateProviderTest, RootCertDistributorSameAsIdentityCertDistributorSameCertNames) { - auto distributor = MakeRefCounted(); - XdsCertificateProvider provider; - provider.UpdateRootCertNameAndDistributor("", "", distributor); - provider.UpdateIdentityCertNameAndDistributor("", "", distributor); + auto root_and_identity_provider = MakeRefCounted(); + auto distributor = root_and_identity_provider->distributor(); + XdsCertificateProvider provider(root_and_identity_provider, "", + root_and_identity_provider, "", {}); auto* watcher = new TestCertificatesWatcher; provider.distributor()->WatchTlsCertificates( std::unique_ptr(watcher), "", ""); @@ -367,186 +387,8 @@ TEST(XdsCertificateProviderTest, EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); } -TEST(XdsCertificateProviderTest, SwapOutDistributorsMultipleTimes) { - auto distributor = MakeRefCounted(); - distributor->SetKeyMaterials("", kRootCert1, MakeKeyCertPairsType1()); - XdsCertificateProvider provider; - auto* watcher = new TestCertificatesWatcher; - provider.distributor()->WatchTlsCertificates( - std::unique_ptr(watcher), "", ""); - // Initially there are no certificate providers. - EXPECT_EQ(watcher->root_certs(), absl::nullopt); - EXPECT_EQ(watcher->key_cert_pairs(), absl::nullopt); - EXPECT_THAT(StatusToString(watcher->root_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for root certificates")); - EXPECT_THAT( - StatusToString(watcher->identity_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for identity certificates")); - // Update root cert distributor. - provider.UpdateRootCertNameAndDistributor("", "", distributor); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), absl::nullopt); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_THAT( - StatusToString(watcher->identity_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for identity certificates")); - // Update identity cert distributor - provider.UpdateIdentityCertNameAndDistributor("", "", distributor); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Update both root and identity certs - distributor->SetKeyMaterials("", kRootCert2, MakeKeyCertPairsType2()); - EXPECT_EQ(watcher->root_certs(), kRootCert2); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Set error for both root and identity - distributor->SetErrorForCert("", GRPC_ERROR_CREATE(kRootErrorMessage), - GRPC_ERROR_CREATE(kIdentityErrorMessage)); - EXPECT_EQ(watcher->root_certs(), kRootCert2); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_THAT(StatusToString(watcher->root_cert_error()), - ::testing::HasSubstr(kRootErrorMessage)); - EXPECT_THAT(StatusToString(watcher->identity_cert_error()), - ::testing::HasSubstr(kIdentityErrorMessage)); - // Send an update again - distributor->SetKeyMaterials("", kRootCert1, MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Remove root cert provider - provider.UpdateRootCertNameAndDistributor("", "", nullptr); - distributor->SetKeyMaterials("", kRootCert2, MakeKeyCertPairsType2()); - EXPECT_EQ(watcher->root_certs(), kRootCert1); // not updated - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_THAT(StatusToString(watcher->root_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for root certificates")); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Remove identity cert provider too - provider.UpdateIdentityCertNameAndDistributor("", "", nullptr); - distributor->SetKeyMaterials("", kRootCert1, MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); // not updated - EXPECT_THAT(StatusToString(watcher->root_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for root certificates")); - EXPECT_THAT( - StatusToString(watcher->identity_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for identity certificates")); - // Change certificate names being watched, without any certificate updates. - provider.UpdateRootCertNameAndDistributor("", "root", distributor); - provider.UpdateIdentityCertNameAndDistributor("", "identity", distributor); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_THAT(StatusToString(watcher->root_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for root certificates")); - EXPECT_THAT( - StatusToString(watcher->identity_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for identity certificates")); - // Send out certificate updates. - distributor->SetKeyMaterials("root", kRootCert2, absl::nullopt); - distributor->SetKeyMaterials("identity", absl::nullopt, - MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_certs(), kRootCert2); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Swap in new certificate distributors with different certificate names and - // existing updates. - auto root_cert_distributor = - MakeRefCounted(); - auto identity_cert_distributor = - MakeRefCounted(); - provider.UpdateRootCertNameAndDistributor("", "root", root_cert_distributor); - provider.UpdateIdentityCertNameAndDistributor("", "identity", - identity_cert_distributor); - EXPECT_EQ(watcher->root_certs(), kRootCert2); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Change certificate names without any certificate updates. - provider.UpdateRootCertNameAndDistributor("", "test", root_cert_distributor); - provider.UpdateIdentityCertNameAndDistributor("", "test", - identity_cert_distributor); - EXPECT_EQ(watcher->root_certs(), kRootCert2); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); - // Send out certificate updates. - root_cert_distributor->SetKeyMaterials("test", kRootCert1, - MakeKeyCertPairsType1()); - identity_cert_distributor->SetKeyMaterials("test", kRootCert2, - MakeKeyCertPairsType2()); - EXPECT_EQ(watcher->root_certs(), kRootCert1); - EXPECT_EQ(watcher->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_EQ(watcher->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher->identity_cert_error(), absl::OkStatus()); -} - -TEST(XdsCertificateProviderTest, MultipleCertNames) { - XdsCertificateProvider provider; - // Start watch for "test1". There are no underlying distributors for - // that cert name, so it will return an error. - auto* watcher1 = new TestCertificatesWatcher; - provider.distributor()->WatchTlsCertificates( - std::unique_ptr(watcher1), "test1", "test1"); - EXPECT_EQ(watcher1->root_certs(), absl::nullopt); - EXPECT_EQ(watcher1->key_cert_pairs(), absl::nullopt); - EXPECT_THAT(StatusToString(watcher1->root_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for root certificates")); - EXPECT_THAT( - StatusToString(watcher1->identity_cert_error()), - ::testing::HasSubstr( - "No certificate provider available for identity certificates")); - // Add distributor for "test1". This will return data to the watcher. - auto cert_distributor1 = MakeRefCounted(); - cert_distributor1->SetKeyMaterials("root", kRootCert1, absl::nullopt); - cert_distributor1->SetKeyMaterials("identity", absl::nullopt, - MakeKeyCertPairsType1()); - provider.UpdateRootCertNameAndDistributor("test1", "root", cert_distributor1); - provider.UpdateIdentityCertNameAndDistributor("test1", "identity", - cert_distributor1); - EXPECT_EQ(watcher1->root_certs(), kRootCert1); - EXPECT_EQ(watcher1->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher1->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher1->identity_cert_error(), absl::OkStatus()); - // Add distributor for "test2". - auto cert_distributor2 = MakeRefCounted(); - cert_distributor2->SetKeyMaterials("root2", kRootCert2, absl::nullopt); - cert_distributor2->SetKeyMaterials("identity2", absl::nullopt, - MakeKeyCertPairsType2()); - provider.UpdateRootCertNameAndDistributor("test2", "root2", - cert_distributor2); - provider.UpdateIdentityCertNameAndDistributor("test2", "identity2", - cert_distributor2); - // Add watcher for "test2". This one should return data immediately. - auto* watcher2 = new TestCertificatesWatcher; - provider.distributor()->WatchTlsCertificates( - std::unique_ptr(watcher2), "test2", "test2"); - EXPECT_EQ(watcher2->root_certs(), kRootCert2); - EXPECT_EQ(watcher2->key_cert_pairs(), MakeKeyCertPairsType2()); - EXPECT_EQ(watcher2->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher2->identity_cert_error(), absl::OkStatus()); - // The presence of "test2" should not affect "test1". - EXPECT_EQ(watcher1->root_certs(), kRootCert1); - EXPECT_EQ(watcher1->key_cert_pairs(), MakeKeyCertPairsType1()); - EXPECT_EQ(watcher1->root_cert_error(), absl::OkStatus()); - EXPECT_EQ(watcher1->identity_cert_error(), absl::OkStatus()); -} - TEST(XdsCertificateProviderTest, UnknownCertName) { - XdsCertificateProvider provider; + XdsCertificateProvider provider(nullptr, "", nullptr, "", {}); auto* watcher = new TestCertificatesWatcher; provider.distributor()->WatchTlsCertificates( std::unique_ptr(watcher), "test", "test"); diff --git a/test/core/xds/xds_cluster_resource_type_test.cc b/test/core/xds/xds_cluster_resource_type_test.cc index 5fd5d20a181..27bb75935fb 100644 --- a/test/core/xds/xds_cluster_resource_type_test.cc +++ b/test/core/xds/xds_cluster_resource_type_test.cc @@ -1347,11 +1347,66 @@ TEST_F(HostOverrideStatusTest, PassesOnRelevantHealthStatuses) { EXPECT_EQ(*decode_result.name, "foo"); auto& resource = static_cast(**decode_result.resource); - EXPECT_THAT(resource.override_host_statuses, - ::testing::UnorderedElementsAre( - XdsHealthStatus(XdsHealthStatus::kUnknown), - XdsHealthStatus(XdsHealthStatus::kHealthy), - XdsHealthStatus(XdsHealthStatus::kDraining))); + EXPECT_EQ(resource.override_host_statuses.ToString(), + "{UNKNOWN, HEALTHY, DRAINING}"); +} + +TEST_F(HostOverrideStatusTest, + DefaultsToUnknownAndHealthyWithoutCommonLbConfig) { + Cluster cluster; + cluster.set_name("foo"); + cluster.set_type(cluster.EDS); + cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self(); + std::string serialized_resource; + ASSERT_TRUE(cluster.SerializeToString(&serialized_resource)); + auto* resource_type = XdsClusterResourceType::Get(); + auto decode_result = + resource_type->Decode(decode_context_, serialized_resource); + ASSERT_TRUE(decode_result.resource.ok()) << decode_result.resource.status(); + ASSERT_TRUE(decode_result.name.has_value()); + EXPECT_EQ(*decode_result.name, "foo"); + auto& resource = + static_cast(**decode_result.resource); + EXPECT_EQ(resource.override_host_statuses.ToString(), "{UNKNOWN, HEALTHY}"); +} + +TEST_F(HostOverrideStatusTest, + DefaultsToUnknownAndHealthyWithoutOverrideHostStatus) { + Cluster cluster; + cluster.set_name("foo"); + cluster.set_type(cluster.EDS); + cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self(); + cluster.mutable_common_lb_config(); + std::string serialized_resource; + ASSERT_TRUE(cluster.SerializeToString(&serialized_resource)); + auto* resource_type = XdsClusterResourceType::Get(); + auto decode_result = + resource_type->Decode(decode_context_, serialized_resource); + ASSERT_TRUE(decode_result.resource.ok()) << decode_result.resource.status(); + ASSERT_TRUE(decode_result.name.has_value()); + EXPECT_EQ(*decode_result.name, "foo"); + auto& resource = + static_cast(**decode_result.resource); + EXPECT_EQ(resource.override_host_statuses.ToString(), "{UNKNOWN, HEALTHY}"); +} + +TEST_F(HostOverrideStatusTest, CanExplicitlySetToEmpty) { + Cluster cluster; + cluster.set_name("foo"); + cluster.set_type(cluster.EDS); + cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self(); + cluster.mutable_common_lb_config()->mutable_override_host_status(); + std::string serialized_resource; + ASSERT_TRUE(cluster.SerializeToString(&serialized_resource)); + auto* resource_type = XdsClusterResourceType::Get(); + auto decode_result = + resource_type->Decode(decode_context_, serialized_resource); + ASSERT_TRUE(decode_result.resource.ok()) << decode_result.resource.status(); + ASSERT_TRUE(decode_result.name.has_value()); + EXPECT_EQ(*decode_result.name, "foo"); + auto& resource = + static_cast(**decode_result.resource); + EXPECT_EQ(resource.override_host_statuses.ToString(), "{}"); } } // namespace diff --git a/test/core/xds/xds_route_config_resource_type_test.cc b/test/core/xds/xds_route_config_resource_type_test.cc index 8bd81fbfcce..a53b5012e7f 100644 --- a/test/core/xds/xds_route_config_resource_type_test.cc +++ b/test/core/xds/xds_route_config_resource_type_test.cc @@ -1749,7 +1749,7 @@ TEST_F(RlsTest, Basic) { ::testing::ElementsAre(::testing::Pair( "rls", "[{\"rls_experimental\":{" - "\"childPolicy\":[{\"cds_experimental\":{}}]," + "\"childPolicy\":[{\"cds_experimental\":{\"isDynamic\":true}}]," "\"childPolicyConfigTargetFieldName\":\"cluster\"," "\"routeLookupConfig\":{" "\"cacheSizeBytes\":\"1024\"," @@ -1852,7 +1852,7 @@ TEST_F(RlsTest, NotUsedInAllVirtualHosts) { ::testing::ElementsAre(::testing::Pair( "rls", "[{\"rls_experimental\":{" - "\"childPolicy\":[{\"cds_experimental\":{}}]," + "\"childPolicy\":[{\"cds_experimental\":{\"isDynamic\":true}}]," "\"childPolicyConfigTargetFieldName\":\"cluster\"," "\"routeLookupConfig\":{" "\"cacheSizeBytes\":\"1024\"," diff --git a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc index c43d8fbc2a0..d4c0ad4a823 100644 --- a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc @@ -330,9 +330,9 @@ TEST_P(CdsDeletionTest, ClusterDeleted) { SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) { if (result.status.ok()) return true; // Keep going. EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code()); - EXPECT_EQ(absl::StrCat("CDS resource \"", kDefaultClusterName, - "\" does not exist"), - result.status.error_message()); + EXPECT_EQ( + absl::StrCat("CDS resource ", kDefaultClusterName, " does not exist"), + result.status.error_message()); return false; }); // Make sure we ACK'ed the update. diff --git a/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc index 3b601e59a8a..4a569d5cfcd 100644 --- a/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc @@ -425,8 +425,8 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) { } // When an RPC fails, we know the channel has seen the update. constexpr char kErrorMessage[] = - "empty address list: DNS resolution failed for server.example.com:443 " - "\\(UNAVAILABLE: injected error\\)"; + "empty address list: DNS resolution failed for server.example.com:443: " + "UNAVAILABLE: injected error"; CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage); // Send an EDS update that moves locality1 to priority 0. args1 = EdsResourceArgs({ @@ -613,7 +613,7 @@ TEST_P(AggregateClusterTest, DependencyLoopWithNoLeafClusters) { cluster_config.add_clusters(kNewClusterName1); custom_cluster->mutable_typed_config()->PackFrom(cluster_config); balancer_->ads_service()->SetCdsResource(cluster); - // kDefaultClusterName points to the default cluster. + // kNewClusterName1 points to the default cluster. cluster.set_name(kNewClusterName1); cluster_config.Clear(); cluster_config.add_clusters(kDefaultClusterName); @@ -621,8 +621,8 @@ TEST_P(AggregateClusterTest, DependencyLoopWithNoLeafClusters) { balancer_->ads_service()->SetCdsResource(cluster); // RPCs should fail. CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, - "new_cluster_1: FAILED_PRECONDITION: " - "aggregate cluster graph has no leaf clusters"); + "aggregate cluster dependency graph for cluster_name " + "has no leaf clusters"); } TEST_P(AggregateClusterTest, DependencyLoopWithLeafClusters) { diff --git a/test/cpp/end2end/xds/xds_core_end2end_test.cc b/test/cpp/end2end/xds/xds_core_end2end_test.cc index 3efcc992daf..6ab8094b5e1 100644 --- a/test/cpp/end2end/xds/xds_core_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_core_end2end_test.cc @@ -429,7 +429,7 @@ TEST_P(TimeoutTest, CdsServerIgnoresRequest) { balancer_->ads_service()->IgnoreResourceType(kCdsTypeUrl); CheckRpcSendFailure( DEBUG_LOCATION, StatusCode::UNAVAILABLE, - absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + absl::StrCat("CDS resource ", kDefaultClusterName, " does not exist"), RpcOptions().set_timeout_ms(4000)); } @@ -437,7 +437,7 @@ TEST_P(TimeoutTest, CdsResourceNotPresentInRequest) { balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); CheckRpcSendFailure( DEBUG_LOCATION, StatusCode::UNAVAILABLE, - absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + absl::StrCat("CDS resource ", kDefaultClusterName, " does not exist"), RpcOptions().set_timeout_ms(4000)); } @@ -461,9 +461,9 @@ TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) { [&](const RpcResult& result) { if (result.status.ok()) return true; // Keep going. EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code()); - EXPECT_EQ(absl::StrCat("CDS resource \"", kNewClusterName, - "\" does not exist"), - result.status.error_message()); + EXPECT_EQ( + absl::StrCat("CDS resource ", kNewClusterName, " does not exist"), + result.status.error_message()); return false; }, /*timeout_ms=*/30000, RpcOptions().set_timeout_ms(4000)); @@ -912,10 +912,9 @@ TEST_P(XdsFederationTest, CdsResourceNameAuthorityUnknown) { grpc::Status status = stub2->Echo(&context, request, &response); EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE); EXPECT_EQ(status.error_message(), - absl::StrCat( - kNewClusterName, - ": UNAVAILABLE: authority \"xds.unknown.com\" not present in " - "bootstrap config")); + absl::StrCat(kNewClusterName, + ": authority \"xds.unknown.com\" not present in " + "bootstrap config")); ASSERT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel2->GetState(false)); } @@ -972,10 +971,10 @@ TEST_P(XdsFederationTest, EdsResourceNameAuthorityUnknown) { EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE); EXPECT_EQ( status.error_message(), - "no children in weighted_target policy: EDS watcher error for resource " + "no children in weighted_target policy: EDS resource " "xdstp://xds.unknown.com/envoy.config.endpoint.v3.ClusterLoadAssignment/" - "edsservice_name (UNAVAILABLE: authority \"xds.unknown.com\" not " - "present in bootstrap config)"); + "edsservice_name: UNAVAILABLE: authority \"xds.unknown.com\" not " + "present in bootstrap config"); ASSERT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel2->GetState(false)); } diff --git a/test/cpp/end2end/xds/xds_csds_end2end_test.cc b/test/cpp/end2end/xds/xds_csds_end2end_test.cc index c99bf4bfcac..575923b41fb 100644 --- a/test/cpp/end2end/xds/xds_csds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_csds_end2end_test.cc @@ -680,7 +680,7 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) { balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); CheckRpcSendFailure( DEBUG_LOCATION, StatusCode::UNAVAILABLE, - absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + absl::StrCat("CDS resource ", kDefaultClusterName, " does not exist"), RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).generic_xds_configs(), diff --git a/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc index 4307b9da28d..e2ab47ac272 100644 --- a/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc @@ -107,6 +107,9 @@ INSTANTIATE_TEST_SUITE_P( // Test to ensure the most basic fault injection config works. TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) { const uint32_t kAbortPercentagePerHundred = 100; + // Create an EDS resource + EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Construct the fault injection filter config HTTPFault http_fault; auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); diff --git a/test/cpp/end2end/xds/xds_routing_end2end_test.cc b/test/cpp/end2end/xds/xds_routing_end2end_test.cc index bb42c8cbcd7..b4b3ec04fb7 100644 --- a/test/cpp/end2end/xds/xds_routing_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_routing_end2end_test.cc @@ -470,6 +470,8 @@ TEST_P(LdsRdsTest, ChooseLastRoute) { } TEST_P(LdsRdsTest, NoMatchingRoute) { + EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); RouteConfiguration route_config = default_route_config_; route_config.mutable_virtual_hosts(0) ->mutable_routes(0) @@ -527,6 +529,8 @@ TEST_P(LdsRdsTest, NacksInvalidRouteConfig) { // Tests that LDS client should fail RPCs with UNAVAILABLE status code if the // matching route has an action other than RouteAction. TEST_P(LdsRdsTest, MatchingRouteHasNoRouteAction) { + EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); RouteConfiguration route_config = default_route_config_; // Set a route with an inappropriate route action auto* vhost = route_config.mutable_virtual_hosts(0); diff --git a/tools/codegen/core/gen_grpc_tls_credentials_options.py b/tools/codegen/core/gen_grpc_tls_credentials_options.py index a7e97f0e214..1c802bee886 100755 --- a/tools/codegen/core/gen_grpc_tls_credentials_options.py +++ b/tools/codegen/core/gen_grpc_tls_credentials_options.py @@ -101,7 +101,7 @@ _DATA_MEMBERS = [ ), test_name="DifferentCertificateVerifier", test_value_1="MakeRefCounted()", - test_value_2='MakeRefCounted(nullptr, "")', + test_value_2="MakeRefCounted(nullptr)", ), DataMember( name="check_call_host", diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index eb1463dcf92..dc859f23350 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1155,7 +1155,6 @@ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h \ src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc \ @@ -1185,8 +1184,12 @@ src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc \ src/core/ext/filters/client_channel/resolver/polling_resolver.cc \ src/core/ext/filters/client_channel/resolver/polling_resolver.h \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h \ src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc \ -src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h \ src/core/ext/filters/client_channel/retry_filter.cc \ src/core/ext/filters/client_channel/retry_filter.h \ src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index e6ba72b8f5c..e3924472666 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -960,7 +960,6 @@ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h \ src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc \ @@ -994,8 +993,12 @@ src/core/ext/filters/client_channel/resolver/polling_resolver.cc \ src/core/ext/filters/client_channel/resolver/polling_resolver.h \ src/core/ext/filters/client_channel/resolver/sockaddr/README.md \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_dependency_manager.h \ src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc \ -src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_attributes.h \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.cc \ +src/core/ext/filters/client_channel/resolver/xds/xds_resolver_trace.h \ src/core/ext/filters/client_channel/retry_filter.cc \ src/core/ext/filters/client_channel/retry_filter.h \ src/core/ext/filters/client_channel/retry_filter_legacy_call_data.cc \