From f8f6034d1a0d3ab1fe2471096546bf699878d49c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 7 Apr 2020 10:03:05 -0700 Subject: [PATCH] Refactor xds LB policy --- BUILD | 105 +- BUILD.gn | 8 +- CMakeLists.txt | 14 +- Makefile | 12 +- build_autogenerated.yaml | 16 +- config.m4 | 8 +- config.w32 | 8 +- doc/environment_variables.md | 5 +- gRPC-C++.podspec | 2 + gRPC-Core.podspec | 8 +- grpc.gemspec | 7 +- grpc.gyp | 14 +- include/grpc/impl/codegen/grpc_types.h | 17 +- package.xml | 7 +- .../lb_policy/address_filtering.cc | 83 + .../lb_policy/address_filtering.h | 99 + .../lb_policy/priority/priority.cc | 875 ++++++++ .../weighted_target/weighted_target.cc | 722 +++++++ .../client_channel/lb_policy/xds/cds.cc | 65 +- .../client_channel/lb_policy/xds/eds.cc | 1172 +++++++++++ .../client_channel/lb_policy/xds/lrs.cc | 524 +++++ .../client_channel/lb_policy/xds/xds.cc | 1754 ----------------- .../client_channel/lb_policy/xds/xds.h | 3 +- .../client_channel/lb_policy_registry.cc | 2 + .../client_channel/xds/xds_client_stats.h | 18 +- .../plugin_registry/grpc_plugin_registry.cc | 20 +- .../grpc_unsecure_plugin_registry.cc | 20 +- src/python/grpcio/grpc_core_dependencies.py | 6 +- .../client_channel/service_config_test.cc | 8 +- test/cpp/end2end/xds_end2end_test.cc | 8 +- tools/doxygen/Doxyfile.c++.internal | 7 +- tools/doxygen/Doxyfile.core.internal | 7 +- .../grpc_xds_bazel_python_test_in_docker.sh | 2 +- .../linux/grpc_xds_bazel_test_in_docker.sh | 2 +- 34 files changed, 3789 insertions(+), 1839 deletions(-) create mode 100644 src/core/ext/filters/client_channel/lb_policy/address_filtering.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/address_filtering.h create mode 100644 src/core/ext/filters/client_channel/lb_policy/priority/priority.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/eds.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc delete mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/xds.cc diff --git a/BUILD b/BUILD index 992d1d403cc..81de8203a03 100644 --- a/BUILD +++ b/BUILD @@ -319,8 +319,9 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_cds", + "grpc_lb_policy_eds", "grpc_lb_policy_grpclb", - "grpc_lb_policy_xds", + "grpc_lb_policy_lrs", "grpc_resolver_xds", ], ) @@ -337,8 +338,9 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_cds_secure", + "grpc_lb_policy_eds_secure", "grpc_lb_policy_grpclb_secure", - "grpc_lb_policy_xds_secure", + "grpc_lb_policy_lrs_secure", "grpc_resolver_xds_secure", "grpc_secure", "grpc_transport_chttp2_client_secure", @@ -1023,7 +1025,9 @@ grpc_cc_library( "grpc_deadline_filter", "grpc_client_authority_filter", "grpc_lb_policy_pick_first", + "grpc_lb_policy_priority", "grpc_lb_policy_round_robin", + "grpc_lb_policy_weighted_target", "grpc_client_idle_filter", "grpc_max_age_filter", "grpc_message_size_filter", @@ -1357,41 +1361,75 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_lb_policy_xds", + name = "grpc_lb_policy_cds", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_xds_client", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_cds_secure", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_xds_client_secure", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_eds", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", ], + external_deps = [ + "absl/strings", + ], language = "c++", deps = [ "grpc_base", "grpc_client_channel", + "grpc_lb_address_filtering", "grpc_xds_client", ], ) grpc_cc_library( - name = "grpc_lb_policy_xds_secure", + name = "grpc_lb_policy_eds_secure", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", ], + external_deps = [ + "absl/strings", + ], language = "c++", deps = [ "grpc_base", "grpc_client_channel", + "grpc_lb_address_filtering", "grpc_xds_client_secure", ], ) grpc_cc_library( - name = "grpc_lb_policy_cds", + name = "grpc_lb_policy_lrs", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", ], language = "c++", deps = [ @@ -1402,9 +1440,9 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_lb_policy_cds_secure", + name = "grpc_lb_policy_lrs_secure", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", ], language = "c++", deps = [ @@ -1414,6 +1452,24 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_lb_address_filtering", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/address_filtering.cc", + ], + hdrs = [ + "src/core/ext/filters/client_channel/lb_policy/address_filtering.h", + ], + external_deps = [ + "absl/strings", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + ], +) + grpc_cc_library( name = "grpc_lb_subchannel_list", hdrs = [ @@ -1452,6 +1508,35 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_lb_policy_priority", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/priority/priority.cc", + ], + external_deps = [ + "absl/strings", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_lb_address_filtering", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_weighted_target", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_lb_address_filtering", + ], +) + grpc_cc_library( name = "lb_server_load_reporting_filter", srcs = [ diff --git a/BUILD.gn b/BUILD.gn index 306f2bfa6d8..129ad2b2639 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -223,6 +223,8 @@ config("grpc_config") { "src/core/ext/filters/client_channel/http_proxy.h", "src/core/ext/filters/client_channel/lb_policy.cc", "src/core/ext/filters/client_channel/lb_policy.h", + "src/core/ext/filters/client_channel/lb_policy/address_filtering.cc", + "src/core/ext/filters/client_channel/lb_policy/address_filtering.h", "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc", "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc", @@ -238,10 +240,13 @@ config("grpc_config") { "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc", + "src/core/ext/filters/client_channel/lb_policy/priority/priority.cc", "src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc", "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h", + "src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc", "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy_factory.h", "src/core/ext/filters/client_channel/lb_policy_registry.cc", @@ -962,6 +967,7 @@ config("grpc_config") { ":address_sorting", ":upb", ":absl/types:optional", + ":absl/strings:strings", ":absl/container:inlined_vector", "//third_party/cares", ":address_sorting", diff --git a/CMakeLists.txt b/CMakeLists.txt index 4496b174ba6..27bf340b65a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1315,6 +1315,7 @@ add_library(grpc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1323,9 +1324,12 @@ add_library(grpc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc src/core/ext/filters/client_channel/lb_policy_registry.cc src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc @@ -1743,6 +1747,7 @@ target_link_libraries(grpc address_sorting upb absl::optional + absl::strings absl::inlined_vector ) if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) @@ -1969,6 +1974,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1977,9 +1983,12 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc src/core/ext/filters/client_channel/lb_policy_registry.cc src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc @@ -2321,6 +2330,7 @@ target_link_libraries(grpc_unsecure address_sorting upb absl::optional + absl::strings absl::inlined_vector ) if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) diff --git a/Makefile b/Makefile index 465c2e3a284..d1a58c87a8b 100644 --- a/Makefile +++ b/Makefile @@ -3640,6 +3640,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -3648,9 +3649,12 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ @@ -4269,6 +4273,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -4277,9 +4282,12 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 9a188b41121..69e0b124799 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -382,6 +382,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.h - src/core/ext/filters/client_channel/http_proxy.h - src/core/ext/filters/client_channel/lb_policy.h + - src/core/ext/filters/client_channel/lb_policy/address_filtering.h - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h @@ -740,6 +741,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.cc - src/core/ext/filters/client_channel/http_proxy.cc - src/core/ext/filters/client_channel/lb_policy.cc + - src/core/ext/filters/client_channel/lb_policy/address_filtering.cc - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -748,9 +750,12 @@ libs: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + - src/core/ext/filters/client_channel/lb_policy/priority/priority.cc - src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + - src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc - src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc - src/core/ext/filters/client_channel/lb_policy_registry.cc - src/core/ext/filters/client_channel/local_subchannel_pool.cc - src/core/ext/filters/client_channel/parse_address.cc @@ -1132,6 +1137,7 @@ libs: - address_sorting - upb - absl/types:optional + - absl/strings:strings - absl/container:inlined_vector baselib: true dll: true @@ -1278,6 +1284,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.h - src/core/ext/filters/client_channel/http_proxy.h - src/core/ext/filters/client_channel/lb_policy.h + - src/core/ext/filters/client_channel/lb_policy/address_filtering.h - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h @@ -1571,6 +1578,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.cc - src/core/ext/filters/client_channel/http_proxy.cc - src/core/ext/filters/client_channel/lb_policy.cc + - src/core/ext/filters/client_channel/lb_policy/address_filtering.cc - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1579,9 +1587,12 @@ libs: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + - src/core/ext/filters/client_channel/lb_policy/priority/priority.cc - src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + - src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc - src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc - src/core/ext/filters/client_channel/lb_policy_registry.cc - src/core/ext/filters/client_channel/local_subchannel_pool.cc - src/core/ext/filters/client_channel/parse_address.cc @@ -1888,6 +1899,7 @@ libs: - address_sorting - upb - absl/types:optional + - absl/strings:strings - absl/container:inlined_vector baselib: true dll: true diff --git a/config.m4 b/config.m4 index 66922ad5527..6b450315f4f 100644 --- a/config.m4 +++ b/config.m4 @@ -50,6 +50,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -58,9 +59,12 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ @@ -821,7 +825,9 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/pick_first) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/priority) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/round_robin) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/weighted_target) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/xds) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/c_ares) diff --git a/config.w32 b/config.w32 index 541cc74b602..59cdbdb1ce4 100644 --- a/config.w32 +++ b/config.w32 @@ -19,6 +19,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\http_connect_handshaker.cc " + "src\\core\\ext\\filters\\client_channel\\http_proxy.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\address_filtering.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\child_policy_handler.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\client_load_reporting_filter.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb.cc " + @@ -27,9 +28,12 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb_client_stats.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\load_balancer_api.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first\\pick_first.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\priority\\priority.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin\\round_robin.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\weighted_target\\weighted_target.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\cds.cc " + - "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\eds.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\lrs.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy_registry.cc " + "src\\core\\ext\\filters\\client_channel\\local_subchannel_pool.cc " + "src\\core\\ext\\filters\\client_channel\\parse_address.cc " + @@ -821,7 +825,9 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\priority"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\weighted_target"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\xds"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver\\dns"); diff --git a/doc/environment_variables.md b/doc/environment_variables.md index e79c390c9a3..ab45f937bff 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -57,6 +57,7 @@ some configuration as environment variables that can be set. - compression - traces compression operations - connectivity_state - traces connectivity state changes to channels - cronet - traces state in the cronet transport engine + - eds_lb - traces eds LB policy - executor - traces grpc's internal thread pool ('the executor') - glb - traces the grpclb load balancer - handshaker - traces handshaking state @@ -66,12 +67,14 @@ some configuration as environment variables that can be set. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - flowctl - traces http2 flow control + - lrs_lb - traces lrs LB policy - op_failure - traces error information when failure is pushed onto a completion queue - pick_first - traces the pick first load balancing policy - plugin_credentials - traces plugin credentials - pollable_refcount - traces reference counting of 'pollable' objects (only in DEBUG) + - priority_lb - traces priority LB policy - resource_quota - trace resource quota objects internals - round_robin - traces the round_robin load balancing policy - queue_pluck @@ -84,8 +87,8 @@ some configuration as environment variables that can be set. - transport_security - traces metadata about secure channel establishment - tcp - traces bytes in and out of a channel - tsi - traces tsi transport security + - weighted_target_lb - traces weighted_target LB policy - xds_client - traces xds client - - xds_lb - traces xds LB policy - xds_resolver - traces xds resolver The following tracers will only run in binaries built in DEBUG mode. This is diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index c8b176b23ea..77980999b59 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -233,6 +233,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', @@ -683,6 +684,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d140b005d7f..749e77b3fa7 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -206,6 +206,8 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', @@ -221,10 +223,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/client_channel/lb_policy/subchannel_list.h', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds.h', 'src/core/ext/filters/client_channel/lb_policy_factory.h', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', @@ -1032,6 +1037,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', diff --git a/grpc.gemspec b/grpc.gemspec index c9a1a8835fd..29e5d3d436b 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -128,6 +128,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/http_proxy.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/address_filtering.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/address_filtering.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc ) @@ -143,10 +145,13 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/priority/priority.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/subchannel_list.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/cds.cc ) - s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/eds.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_factory.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_registry.cc ) diff --git a/grpc.gyp b/grpc.gyp index b065ee6f2af..a62c034d65a 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -426,6 +426,7 @@ 'address_sorting', 'upb', 'absl/types:optional', + 'absl/strings:strings', 'absl/container:inlined_vector', ], 'sources': [ @@ -442,6 +443,7 @@ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -450,9 +452,12 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', @@ -916,6 +921,7 @@ 'address_sorting', 'upb', 'absl/types:optional', + 'absl/strings:strings', 'absl/container:inlined_vector', ], 'sources': [ @@ -932,6 +938,7 @@ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -940,9 +947,12 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index ab4c39f9310..ff45450f3a3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -348,18 +348,11 @@ typedef struct { balancer before using fallback backend addresses from the resolver. If 0, enter fallback mode immediately. Default value is 10000. */ #define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms" -/* Time in milliseconds to wait before a locality is deleted after it's removed - from the received EDS update. If 0, delete the locality immediately. Default - value is 15 minutes. */ -#define GRPC_ARG_LOCALITY_RETENTION_INTERVAL_MS \ - "grpc.xds_locality_retention_interval_ms" -/* Timeout in milliseconds to wait for the localities of a specific priority to - complete their initial connection attempt before xDS fails over to the next - priority. Specifically, the connection attempt of a priority is considered - completed when any locality of that priority is ready or all the localities - of that priority fail to connect. If 0, failover happens immediately. Default - value is 10 seconds. */ -#define GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS "grpc.xds_failover_timeout_ms" +/* Timeout in milliseconds to wait for the child of a specific priority to + complete its initial connection attempt before the priority LB policy fails + over to the next priority. Default value is 10 seconds. */ +#define GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS \ + "grpc.priority_failover_timeout_ms" /* Timeout in milliseconds to wait for a resource to be returned from * the xds server before assuming that it does not exist. * The default is 15 seconds. */ diff --git a/package.xml b/package.xml index 32930b6f65c..06d1669ec48 100644 --- a/package.xml +++ b/package.xml @@ -108,6 +108,8 @@ + + @@ -123,10 +125,13 @@ + + - + + diff --git a/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc b/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc new file mode 100644 index 00000000000..67843df78e5 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc @@ -0,0 +1,83 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" + +#include "src/core/lib/channel/channel_args.h" + +#define GRPC_ARG_HIERARCHICAL_PATH "grpc.internal.address.hierarchical_path" + +namespace grpc_core { + +namespace { + +void* HierarchicalPathCopy(void* p) { + std::vector* path = static_cast*>(p); + return static_cast(new std::vector(*path)); +} + +void HierarchicalPathDestroy(void* p) { + std::vector* path = static_cast*>(p); + delete path; +} + +int HierarchicalPathCompare(void* p1, void* p2) { + std::vector* path1 = static_cast*>(p1); + std::vector* path2 = static_cast*>(p2); + for (size_t i = 0; i < path1->size(); ++i) { + if (path2->size() == i) return 1; + int r = (*path1)[i].compare((*path2)[i]); + if (r != 0) return r; + } + if (path2->size() > path1->size()) return -1; + return 0; +} + +const grpc_arg_pointer_vtable hierarchical_path_arg_vtable = { + HierarchicalPathCopy, HierarchicalPathDestroy, HierarchicalPathCompare}; + +} // namespace + +grpc_arg MakeHierarchicalPathArg(const std::vector& path) { + return grpc_channel_arg_pointer_create( + const_cast(GRPC_ARG_HIERARCHICAL_PATH), + const_cast*>(&path), + &hierarchical_path_arg_vtable); +} + +HierarchicalAddressMap MakeHierarchicalAddressMap( + const ServerAddressList& addresses) { + HierarchicalAddressMap result; + for (const ServerAddress& address : addresses) { + auto* path = grpc_channel_args_find_pointer>( + address.args(), GRPC_ARG_HIERARCHICAL_PATH); + if (path == nullptr || path->empty()) continue; + auto it = path->begin(); + ServerAddressList& target_list = result[*it]; + ++it; + std::vector remaining_path(it, path->end()); + const char* name_to_remove = GRPC_ARG_HIERARCHICAL_PATH; + grpc_arg new_arg = MakeHierarchicalPathArg(remaining_path); + grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove( + address.args(), &name_to_remove, 1, &new_arg, 1); + target_list.emplace_back(address.address(), new_args); + } + return result; +} + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/lb_policy/address_filtering.h b/src/core/ext/filters/client_channel/lb_policy/address_filtering.h new file mode 100644 index 00000000000..03a1c228e7a --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/address_filtering.h @@ -0,0 +1,99 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H + +#include + +#include +#include +#include + +#include "absl/strings/string_view.h" + +#include "src/core/ext/filters/client_channel/server_address.h" + +// The resolver returns a flat list of addresses. When a hierarchy of +// LB policies is in use, each leaf of the hierarchy will need a +// different subset of those addresses. This library provides a +// mechanism for determining which address is passed to which leaf +// policy. +// +// Each address will have an associated path that indicates which child +// it should be sent to at each level of the hierarchy to wind up at the +// right leaf policy. Each LB policy will look at the first element of +// the path of each address to determine which child to send the address +// to. It will then remove that first element when passing the address +// down to its child. +// +// For example, consider the following LB policy hierarchy: +// +// - priority +// - child0 (weighted_target) +// - localityA (round_robin) +// - localityB (round_robin) +// - child1 (weighted_target) +// - localityC (round_robin) +// - localityD (round_robin) +// +// Now consider the following addresses: +// - 10.0.0.1:80 path=["child0", "localityA"] +// - 10.0.0.2:80 path=["child0", "localityB"] +// - 10.0.0.3:80 path=["child1", "localityC"] +// - 10.0.0.4:80 path=["child1", "localityD"] +// +// The priority policy will split this up into two lists, one for each +// of its children: +// - child0: +// - 10.0.0.1:80 path=["localityA"] +// - 10.0.0.2:80 path=["localityB"] +// - child1: +// - 10.0.0.3:80 path=["localityC"] +// - 10.0.0.4:80 path=["localityD"] +// +// The weighted_target policy for child0 will split its list up into two +// lists, one for each of its children: +// - localityA: +// - 10.0.0.1:80 path=[] +// - localityB: +// - 10.0.0.2:80 path=[] +// +// Similarly, the weighted_target policy for child1 will split its list +// up into two lists, one for each of its children: +// - localityC: +// - 10.0.0.3:80 path=[] +// - localityD: +// - 10.0.0.4:80 path=[] + +namespace grpc_core { + +// Constructs a channel arg containing the hierarchical path +// to be associated with an address. +grpc_arg MakeHierarchicalPathArg(const std::vector& path); + +// A map from the next path element to the addresses that fall under +// that path element. +using HierarchicalAddressMap = std::map; + +// Splits up the addresses into a separate list for each child. +HierarchicalAddressMap MakeHierarchicalAddressMap( + const ServerAddressList& addresses); + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H \ + */ diff --git a/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc b/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc new file mode 100644 index 00000000000..ff69112387c --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc @@ -0,0 +1,875 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include + +#include "absl/strings/str_cat.h" + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" + +namespace grpc_core { + +TraceFlag grpc_lb_priority_trace(false, "priority_lb"); + +namespace { + +constexpr char kPriority[] = "priority_experimental"; + +// How long we keep a child around for after it is no longer being used +// (either because it has been removed from the config or because we +// have switched to a higher-priority child). +constexpr int kChildRetentionIntervalMs = 15 * 60 * 1000; + +// Default for how long we wait for a newly created child to get connected +// before starting to attempt the next priority. Overridable via channel arg. +constexpr int kDefaultChildFailoverTimeoutMs = 10000; + +// Config for priority LB policy. +class PriorityLbConfig : public LoadBalancingPolicy::Config { + public: + PriorityLbConfig( + std::map> + children, + std::vector priorities) + : children_(std::move(children)), priorities_(std::move(priorities)) {} + + const char* name() const override { return kPriority; } + + const std::map>& + children() const { + return children_; + } + const std::vector& priorities() const { return priorities_; } + + private: + const std::map> + children_; + const std::vector priorities_; +}; + +// priority LB policy. +class PriorityLb : public LoadBalancingPolicy { + public: + explicit PriorityLb(Args args); + + const char* name() const override { return kPriority; } + + void UpdateLocked(UpdateArgs args) override; + void ExitIdleLocked() override; + void ResetBackoffLocked() override; + + private: + // Each ChildPriority holds a ref to the PriorityLb. + class ChildPriority : public InternallyRefCounted { + public: + ChildPriority(RefCountedPtr priority_policy, std::string name); + + ~ChildPriority() { + priority_policy_.reset(DEBUG_LOCATION, "ChildPriority"); + } + + const std::string& name() const { return name_; } + + void UpdateLocked(RefCountedPtr config); + void ExitIdleLocked(); + void ResetBackoffLocked(); + void DeactivateLocked(); + void MaybeReactivateLocked(); + void MaybeCancelFailoverTimerLocked(); + + void Orphan() override; + + std::unique_ptr GetPicker() { + return absl::make_unique(picker_wrapper_); + } + + grpc_connectivity_state connectivity_state() const { + return connectivity_state_; + } + bool failover_timer_callback_pending() const { + return failover_timer_callback_pending_; + } + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class RefCountedPicker : public RefCounted { + public: + explicit RefCountedPicker(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A non-ref-counted wrapper for RefCountedPicker. + class RefCountedPickerWrapper : public SubchannelPicker { + public: + explicit RefCountedPickerWrapper(RefCountedPtr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) override { return picker_->Pick(args); } + + private: + RefCountedPtr picker_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr priority) + : priority_(std::move(priority)) {} + + ~Helper() { priority_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr priority_; + }; + + // Methods for dealing with the child policy. + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + + void OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, + std::unique_ptr picker); + + void StartFailoverTimerLocked(); + + static void OnFailoverTimer(void* arg, grpc_error* error); + static void OnFailoverTimerLocked(void* arg, grpc_error* error); + static void OnDeactivationTimer(void* arg, grpc_error* error); + static void OnDeactivationTimerLocked(void* arg, grpc_error* error); + + RefCountedPtr priority_policy_; + const std::string name_; + + OrphanablePtr child_policy_; + + grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_CONNECTING; + RefCountedPtr picker_wrapper_; + + // States for delayed removal. + grpc_timer deactivation_timer_; + grpc_closure on_deactivation_timer_; + bool deactivation_timer_callback_pending_ = false; + + // States of failover. + grpc_timer failover_timer_; + grpc_closure on_failover_timer_; + grpc_closure on_failover_timer_locked_; + bool failover_timer_callback_pending_ = false; + }; + + ~PriorityLb(); + + void ShutdownLocked() override; + + // Returns UINT32_MAX if child is not in current priority list. + uint32_t GetChildPriorityLocked(const std::string& child_name) const; + + void HandleChildConnectivityStateChangeLocked(ChildPriority* child); + void DeleteChild(ChildPriority* child); + + void TryNextPriorityLocked(bool report_connecting); + void SelectPriorityLocked(uint32_t priority); + + const int child_failover_timeout_ms_; + + // Current channel args and config from the resolver. + const grpc_channel_args* args_ = nullptr; + RefCountedPtr config_; + HierarchicalAddressMap addresses_; + + // Internal state. + bool shutting_down_ = false; + + std::map> children_; + // The priority that is being used. + uint32_t current_priority_ = UINT32_MAX; + // Points to the current child from before the most recent update. + // We will continue to use this child until we decide which of the new + // children to use. + ChildPriority* current_child_from_before_update_ = nullptr; +}; + +// +// PriorityLb +// + +PriorityLb::PriorityLb(Args args) + : LoadBalancingPolicy(std::move(args)), + child_failover_timeout_ms_(grpc_channel_args_find_integer( + args.args, GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, + {kDefaultChildFailoverTimeoutMs, 0, INT_MAX})) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] created", this); + } +} + +PriorityLb::~PriorityLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] destroying priority LB policy", this); + } + grpc_channel_args_destroy(args_); +} + +void PriorityLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] shutting down", this); + } + shutting_down_ = true; + children_.clear(); +} + +void PriorityLb::ExitIdleLocked() { + if (current_priority_ != UINT32_MAX) { + const std::string& child_name = config_->priorities()[current_priority_]; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] exiting IDLE for current priority %d child %s", + this, current_priority_, child_name.c_str()); + } + children_[child_name]->ExitIdleLocked(); + } +} + +void PriorityLb::ResetBackoffLocked() { + for (const auto& p : children_) p.second->ResetBackoffLocked(); +} + +void PriorityLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] received update", this); + } + // Save current child. + if (current_priority_ != UINT32_MAX) { + const std::string& child_name = config_->priorities()[current_priority_]; + current_child_from_before_update_ = children_[child_name].get(); + // Unset current_priority_, since it was an index into the old + // config's priority list and may no longer be valid. It will be + // reset later by TryNextPriorityLocked(), but we unset it here in + // case updating any of our children triggers a state update. + current_priority_ = UINT32_MAX; + } + // Update config. + config_ = std::move(args.config); + // Update args. + grpc_channel_args_destroy(args_); + args_ = args.args; + args.args = nullptr; + // Update addresses. + addresses_ = MakeHierarchicalAddressMap(args.addresses); + // Check all existing children against the new config. + for (const auto& p : children_) { + const std::string& child_name = p.first; + auto& child = p.second; + auto config_it = config_->children().find(child_name); + if (config_it == config_->children().end()) { + // Existing child not found in new config. Deactivate it. + child->DeactivateLocked(); + } else { + // Existing child found in new config. Update it. + child->UpdateLocked(config_it->second); + } + } + // Try to get connected. + TryNextPriorityLocked(/*report_connecting=*/children_.empty()); +} + +uint32_t PriorityLb::GetChildPriorityLocked( + const std::string& child_name) const { + for (uint32_t priority = 0; priority < config_->priorities().size(); + ++priority) { + if (config_->priorities()[priority] == child_name) return priority; + } + return UINT32_MAX; +} + +void PriorityLb::HandleChildConnectivityStateChangeLocked( + ChildPriority* child) { + // Special case for the child that was the current child before the + // most recent update. + if (child == current_child_from_before_update_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] state update for current child from before " + "config update", + this); + } + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + // If it's still READY or IDLE, we stick with this child, so pass + // the new picker up to our parent. + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); + } else { + // If it's no longer READY or IDLE, we should stop using it. + // We already started trying other priorities as a result of the + // update, but calling TryNextPriorityLocked() ensures that we will + // properly select between CONNECTING and TRANSIENT_FAILURE as the + // new state to report to our parent. + current_child_from_before_update_ = nullptr; + TryNextPriorityLocked(/*report_connecting=*/true); + } + return; + } + // Otherwise, find the child's priority. + uint32_t child_priority = GetChildPriorityLocked(child->name()); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] state update for priority %d, child %s", + this, child_priority, child->name().c_str()); + } + // Ignore priorities not in the current config. + if (child_priority == UINT32_MAX) return; + // Ignore lower-than-current priorities. + if (child_priority > current_priority_) return; + // If a child reports TRANSIENT_FAILURE, start trying the next priority. + // Note that even if this is for a higher-than-current priority, we + // may still need to create some children between this priority and + // the current one (e.g., if we got an update that inserted new + // priorities ahead of the current one). + if (child->connectivity_state() == GRPC_CHANNEL_TRANSIENT_FAILURE) { + TryNextPriorityLocked( + /*report_connecting=*/child_priority == current_priority_); + return; + } + // The update is for a higher-than-current priority (or for any + // priority if we don't have any current priority). + if (child_priority < current_priority_) { + // If the child reports READY or IDLE, switch to that priority. + // Otherwise, ignore the update. + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + SelectPriorityLocked(child_priority); + } + return; + } + // The current priority has returned a new picker, so pass it up to + // our parent. + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); +} + +void PriorityLb::DeleteChild(ChildPriority* child) { + // If this was the current child from before the most recent update, + // stop using it. We already started trying other priorities as a + // result of the update, but calling TryNextPriorityLocked() ensures that + // we will properly select between CONNECTING and TRANSIENT_FAILURE as the + // new state to report to our parent. + if (current_child_from_before_update_ == child) { + current_child_from_before_update_ = nullptr; + TryNextPriorityLocked(/*report_connecting=*/true); + } + children_.erase(child->name()); +} + +void PriorityLb::TryNextPriorityLocked(bool report_connecting) { + for (uint32_t priority = 0; priority < config_->priorities().size(); + ++priority) { + // If the child for the priority does not exist yet, create it. + const std::string& child_name = config_->priorities()[priority]; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] trying priority %d, child %s", this, + priority, child_name.c_str()); + } + auto& child = children_[child_name]; + if (child == nullptr) { + if (report_connecting) { + channel_control_helper()->UpdateState( + GRPC_CHANNEL_CONNECTING, + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker"))); + } + child = MakeOrphanable( + Ref(DEBUG_LOCATION, "ChildPriority"), child_name); + child->UpdateLocked(config_->children().find(child_name)->second); + return; + } + // The child already exists. + child->MaybeReactivateLocked(); + // If the child is in state READY or IDLE, switch to it. + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + SelectPriorityLocked(priority); + return; + } + // Child is not READY or IDLE. + // If its failover timer is still pending, give it time to fire. + if (child->failover_timer_callback_pending()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] priority %d, child %s: child still " + "attempting to connect, will wait", + this, priority, child_name.c_str()); + } + if (report_connecting) { + channel_control_helper()->UpdateState( + GRPC_CHANNEL_CONNECTING, + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker"))); + } + return; + } + // Child has been failing for a while. Move on to the next priority. + } + // If there are no more priorities to try, report TRANSIENT_FAILURE. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] no priority reachable, putting channel in " + "TRANSIENT_FAILURE", + this); + } + current_priority_ = UINT32_MAX; + current_child_from_before_update_ = nullptr; + grpc_error* error = grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("no ready priority"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, + absl::make_unique(error)); +} + +void PriorityLb::SelectPriorityLocked(uint32_t priority) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] selected priority %d, child %s", this, + priority, config_->priorities()[priority].c_str()); + } + current_priority_ = priority; + current_child_from_before_update_ = nullptr; + // Deactivate lower priorities. + for (uint32_t p = priority + 1; p < config_->priorities().size(); ++p) { + const std::string& child_name = config_->priorities()[p]; + auto it = children_.find(child_name); + if (it != children_.end()) it->second->DeactivateLocked(); + } + // Update picker. + auto& child = children_[config_->priorities()[priority]]; + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); +} + +// +// PriorityLb::ChildPriority +// + +PriorityLb::ChildPriority::ChildPriority( + RefCountedPtr priority_policy, std::string name) + : priority_policy_(std::move(priority_policy)), name_(std::move(name)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] creating child %s (%p)", + priority_policy_.get(), name_.c_str(), this); + } + GRPC_CLOSURE_INIT(&on_failover_timer_, OnFailoverTimer, this, + grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&on_failover_timer_locked_, OnFailoverTimerLocked, this, + nullptr); + // Start the failover timer. + StartFailoverTimerLocked(); +} + +void PriorityLb::ChildPriority::Orphan() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): orphaned", + priority_policy_.get(), name_.c_str(), this); + } + MaybeCancelFailoverTimerLocked(); + if (deactivation_timer_callback_pending_) { + grpc_timer_cancel(&deactivation_timer_); + } + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + priority_policy_->interested_parties()); + child_policy_.reset(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_wrapper_.reset(); + if (deactivation_timer_callback_pending_) { + grpc_timer_cancel(&deactivation_timer_); + } + Unref(DEBUG_LOCATION, "ChildPriority+Orphan"); +} + +void PriorityLb::ChildPriority::UpdateLocked( + RefCountedPtr config) { + if (priority_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): start update", + priority_policy_.get(), name_.c_str(), this); + } + // Create policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(priority_policy_->args_); + } + // Construct update args. + UpdateArgs update_args; + update_args.config = std::move(config); + update_args.addresses = priority_policy_->addresses_[name_]; + update_args.args = grpc_channel_args_copy(priority_policy_->args_); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): updating child policy handler %p", + priority_policy_.get(), name_.c_str(), this, child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +OrphanablePtr +PriorityLb::ChildPriority::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = priority_policy_->combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_priority_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): created new child policy " + "handler %p", + priority_policy_.get(), name_.c_str(), this, lb_policy.get()); + } + // Add the parent's interested_parties pollset_set to that of the newly + // created child policy. This will make the child policy progress upon + // activity on the parent LB, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + priority_policy_->interested_parties()); + return lb_policy; +} + +void PriorityLb::ChildPriority::ExitIdleLocked() { + if (connectivity_state_ == GRPC_CHANNEL_IDLE && + !failover_timer_callback_pending_) { + StartFailoverTimerLocked(); + } + child_policy_->ExitIdleLocked(); +} + +void PriorityLb::ChildPriority::ResetBackoffLocked() { + child_policy_->ResetBackoffLocked(); +} + +void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, std::unique_ptr picker) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): state update: %s, picker %p", + priority_policy_.get(), name_.c_str(), this, + ConnectivityStateName(state), picker.get()); + } + // Store the state and picker. + connectivity_state_ = state; + picker_wrapper_ = MakeRefCounted(std::move(picker)); + // If READY or TRANSIENT_FAILURE, cancel failover timer. + if (state == GRPC_CHANNEL_READY || state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + MaybeCancelFailoverTimerLocked(); + } + // Notify the parent policy. + priority_policy_->HandleChildConnectivityStateChangeLocked(this); +} + +void PriorityLb::ChildPriority::StartFailoverTimerLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): starting failover timer for %d ms", + priority_policy_.get(), name_.c_str(), this, + priority_policy_->child_failover_timeout_ms_); + } + Ref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked").release(); + grpc_timer_init( + &failover_timer_, + ExecCtx::Get()->Now() + priority_policy_->child_failover_timeout_ms_, + &on_failover_timer_); + failover_timer_callback_pending_ = true; +} + +void PriorityLb::ChildPriority::MaybeCancelFailoverTimerLocked() { + if (failover_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): cancelling failover timer", + priority_policy_.get(), name_.c_str(), this); + } + grpc_timer_cancel(&failover_timer_); + failover_timer_callback_pending_ = false; + } +} + +void PriorityLb::ChildPriority::OnFailoverTimer(void* arg, grpc_error* error) { + ChildPriority* self = static_cast(arg); + self->priority_policy_->combiner()->Run(&self->on_failover_timer_locked_, + GRPC_ERROR_REF(error)); +} + +void PriorityLb::ChildPriority::OnFailoverTimerLocked(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && self->failover_timer_callback_pending_ && + !self->priority_policy_->shutting_down_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): failover timer fired, " + "reporting TRANSIENT_FAILURE", + self->priority_policy_.get(), self->name_.c_str(), self); + } + self->failover_timer_callback_pending_ = false; + self->OnConnectivityStateUpdateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE, + nullptr); + } + self->Unref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked"); +} + +void PriorityLb::ChildPriority::DeactivateLocked() { + // If already deactivated, don't do it again. + if (deactivation_timer_callback_pending_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): deactivating -- will remove in %d " + "ms.", + priority_policy_.get(), name_.c_str(), this, + kChildRetentionIntervalMs); + } + MaybeCancelFailoverTimerLocked(); + // Start a timer to delete the child. + Ref(DEBUG_LOCATION, "ChildPriority+timer").release(); + GRPC_CLOSURE_INIT(&on_deactivation_timer_, OnDeactivationTimer, this, + grpc_schedule_on_exec_ctx); + grpc_timer_init(&deactivation_timer_, + ExecCtx::Get()->Now() + kChildRetentionIntervalMs, + &on_deactivation_timer_); + deactivation_timer_callback_pending_ = true; +} + +void PriorityLb::ChildPriority::MaybeReactivateLocked() { + if (deactivation_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): reactivating", + priority_policy_.get(), name_.c_str(), this); + } + deactivation_timer_callback_pending_ = false; + grpc_timer_cancel(&deactivation_timer_); + } +} + +void PriorityLb::ChildPriority::OnDeactivationTimer(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + self->priority_policy_->combiner()->Run( + GRPC_CLOSURE_INIT(&self->on_deactivation_timer_, + OnDeactivationTimerLocked, self, nullptr), + GRPC_ERROR_REF(error)); +} + +void PriorityLb::ChildPriority::OnDeactivationTimerLocked(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && self->deactivation_timer_callback_pending_ && + !self->priority_policy_->shutting_down_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): deactivation timer fired, " + "deleting child", + self->priority_policy_.get(), self->name_.c_str(), self); + } + self->deactivation_timer_callback_pending_ = false; + self->priority_policy_->DeleteChild(self); + } + self->Unref(DEBUG_LOCATION, "ChildPriority+timer"); +} + +// +// PriorityLb::ChildPriority::Helper +// + +void PriorityLb::ChildPriority::Helper::RequestReresolution() { + if (priority_->priority_policy_->shutting_down_) return; + priority_->priority_policy_->channel_control_helper()->RequestReresolution(); +} + +RefCountedPtr +PriorityLb::ChildPriority::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (priority_->priority_policy_->shutting_down_) return nullptr; + return priority_->priority_policy_->channel_control_helper() + ->CreateSubchannel(args); +} + +void PriorityLb::ChildPriority::Helper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (priority_->priority_policy_->shutting_down_) return; + // Notify the priority. + priority_->OnConnectivityStateUpdateLocked(state, std::move(picker)); +} + +void PriorityLb::ChildPriority::Helper::AddTraceEvent(TraceSeverity severity, + StringView message) { + if (priority_->priority_policy_->shutting_down_) return; + priority_->priority_policy_->channel_control_helper()->AddTraceEvent(severity, + message); +} + +// +// factory +// + +class PriorityLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args)); + } + + const char* name() const override { return kPriority; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // priority was mentioned as a policy in the deprecated + // loadBalancingPolicy field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:priority policy requires " + "configuration. Please use loadBalancingConfig field of service " + "config instead."); + return nullptr; + } + std::vector error_list; + // Children. + std::map> children; + auto it = json.object_value().find("children"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:children error:required field missing")); + } else if (it->second.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:children error:type should be object")); + } else { + const Json::Object& object = it->second.object_value(); + for (const auto& p : object) { + const std::string& child_name = p.first; + const Json& element = p.second; + if (element.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name, + " error:should be type object") + .c_str())); + } else { + auto it2 = element.object_value().find("config"); + if (it2 == element.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name, + " error:missing 'config' field") + .c_str())); + } else { + grpc_error* parse_error = GRPC_ERROR_NONE; + auto config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + it2->second, &parse_error); + if (config == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back( + GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name).c_str(), + &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + children[child_name] = std::move(config); + } + } + } + } + // Priorities. + std::vector priorities; + it = json.object_value().find("priorities"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:priorities error:required field missing")); + } else if (it->second.type() != Json::Type::ARRAY) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:priorities error:type should be array")); + } else { + const Json::Array& array = it->second.array_value(); + for (size_t i = 0; i < array.size(); ++i) { + const Json& element = array[i]; + if (element.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities element:", i, + " error:should be type string") + .c_str())); + } else if (children.find(element.string_value()) == children.end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities element:", i, + " error:unknown child '", element.string_value(), + "'") + .c_str())); + } else { + priorities.emplace_back(element.string_value()); + } + } + if (priorities.size() != children.size()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities error:priorities size (", + priorities.size(), ") != children size (", + children.size(), ")") + .c_str())); + } + } + if (error_list.empty()) { + return MakeRefCounted(std::move(children), + std::move(priorities)); + } else { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "priority_experimental LB policy config", &error_list); + return nullptr; + } + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_priority_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_priority_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc new file mode 100644 index 00000000000..a319c40cd31 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc @@ -0,0 +1,722 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include +#include + +#include "absl/strings/str_cat.h" + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" + +namespace grpc_core { + +TraceFlag grpc_lb_weighted_target_trace(false, "weighted_target_lb"); + +namespace { + +constexpr char kWeightedTarget[] = "weighted_target_experimental"; + +// How long we keep a child around for after it has been removed from +// the config. +constexpr int kChildRetentionIntervalMs = 15 * 60 * 1000; + +// Config for weighted_target LB policy. +class WeightedTargetLbConfig : public LoadBalancingPolicy::Config { + public: + struct ChildConfig { + uint32_t weight; + RefCountedPtr config; + }; + + using TargetMap = std::map; + + explicit WeightedTargetLbConfig(TargetMap target_map) + : target_map_(std::move(target_map)) {} + + const char* name() const override { return kWeightedTarget; } + + const TargetMap& target_map() const { return target_map_; } + + private: + TargetMap target_map_; +}; + +// weighted_target LB policy. +class WeightedTargetLb : public LoadBalancingPolicy { + public: + explicit WeightedTargetLb(Args args); + + const char* name() const override { return kWeightedTarget; } + + void UpdateLocked(UpdateArgs args) override; + void ResetBackoffLocked() override; + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class ChildPickerWrapper : public RefCounted { + public: + explicit ChildPickerWrapper(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // Picks a child using stateless WRR and then delegates to that + // child's picker. + class WeightedPicker : public SubchannelPicker { + public: + // Maintains a weighted list of pickers from each child that is in + // ready state. The first element in the pair represents the end of a + // range proportional to the child's weight. The start of the range + // is the previous value in the vector and is 0 for the first element. + using PickerList = + InlinedVector>, + 1>; + + explicit WeightedPicker(PickerList pickers) + : pickers_(std::move(pickers)) {} + + PickResult Pick(PickArgs args) override; + + private: + PickerList pickers_; + }; + + // Each WeightedChild holds a ref to its parent WeightedTargetLb. + class WeightedChild : public InternallyRefCounted { + public: + WeightedChild(RefCountedPtr weighted_target_policy, + const std::string& name); + ~WeightedChild(); + + void Orphan() override; + + void UpdateLocked(const WeightedTargetLbConfig::ChildConfig& config, + ServerAddressList addresses, + const grpc_channel_args* args); + void ResetBackoffLocked(); + void DeactivateLocked(); + + uint32_t weight() const { return weight_; } + grpc_connectivity_state connectivity_state() const { + return connectivity_state_; + } + RefCountedPtr picker_wrapper() const { + return picker_wrapper_; + } + + private: + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr weighted_child) + : weighted_child_(std::move(weighted_child)) {} + + ~Helper() { weighted_child_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr weighted_child_; + }; + + // Methods for dealing with the child policy. + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + + void OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, + std::unique_ptr picker); + + static void OnDelayedRemovalTimer(void* arg, grpc_error* error); + static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); + + // The owning LB policy. + RefCountedPtr weighted_target_policy_; + + const std::string& name_; + + uint32_t weight_; + + OrphanablePtr child_policy_; + + RefCountedPtr picker_wrapper_; + grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_CONNECTING; + bool seen_failure_since_ready_ = false; + + // States for delayed removal. + grpc_timer delayed_removal_timer_; + grpc_closure on_delayed_removal_timer_; + bool delayed_removal_timer_callback_pending_ = false; + bool shutdown_ = false; + }; + + ~WeightedTargetLb(); + + void ShutdownLocked() override; + + void UpdateStateLocked(); + + // Current config from the resolver. + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // Children. + std::map> targets_; +}; + +// +// WeightedTargetLb::WeightedPicker +// + +WeightedTargetLb::PickResult WeightedTargetLb::WeightedPicker::Pick( + PickArgs args) { + // Generate a random number in [0, total weight). + const uint32_t key = rand() % pickers_[pickers_.size() - 1].first; + // Find the index in pickers_ corresponding to key. + size_t mid = 0; + size_t start_index = 0; + size_t end_index = pickers_.size() - 1; + size_t index = 0; + while (end_index > start_index) { + mid = (start_index + end_index) / 2; + if (pickers_[mid].first > key) { + end_index = mid; + } else if (pickers_[mid].first < key) { + start_index = mid + 1; + } else { + index = mid + 1; + break; + } + } + if (index == 0) index = start_index; + GPR_ASSERT(pickers_[index].first > key); + // Delegate to the child picker. + return pickers_[index].second->Pick(args); +} + +// +// WeightedTargetLb +// + +WeightedTargetLb::WeightedTargetLb(Args args) + : LoadBalancingPolicy(std::move(args)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] created", this); + } +} + +WeightedTargetLb::~WeightedTargetLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] destroying weighted_target LB policy", + this); + } +} + +void WeightedTargetLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] shutting down", this); + } + shutting_down_ = true; + targets_.clear(); +} + +void WeightedTargetLb::ResetBackoffLocked() { + for (auto& p : targets_) p.second->ResetBackoffLocked(); +} + +void WeightedTargetLb::UpdateLocked(UpdateArgs args) { + if (shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] Received update", this); + } + // Update config. + config_ = std::move(args.config); + // Deactivate the targets not in the new config. + for (const auto& p : targets_) { + const std::string& name = p.first; + WeightedChild* child = p.second.get(); + if (config_->target_map().find(name) == config_->target_map().end()) { + child->DeactivateLocked(); + } + } + // Add or update the targets in the new config. + HierarchicalAddressMap address_map = + MakeHierarchicalAddressMap(args.addresses); + for (const auto& p : config_->target_map()) { + const std::string& name = p.first; + const WeightedTargetLbConfig::ChildConfig& config = p.second; + auto it = targets_.find(name); + if (it == targets_.end()) { + it = targets_.emplace(std::make_pair(name, nullptr)).first; + it->second = MakeOrphanable( + Ref(DEBUG_LOCATION, "WeightedChild"), it->first); + } + it->second->UpdateLocked(config, std::move(address_map[name]), args.args); + } +} + +void WeightedTargetLb::UpdateStateLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] scanning children to determine " + "connectivity state", + this); + } + // Construct a new picker which maintains a map of all child pickers + // that are ready. Each child is represented by a portion of the range + // proportional to its weight, such that the total range is the sum of the + // weights of all children. + WeightedPicker::PickerList picker_list; + uint32_t end = 0; + // Also count the number of children in each state, to determine the + // overall state. + size_t num_connecting = 0; + size_t num_idle = 0; + size_t num_transient_failures = 0; + for (const auto& p : targets_) { + const std::string& child_name = p.first; + const WeightedChild* child = p.second.get(); + // Skip the targets that are not in the latest update. + if (config_->target_map().find(child_name) == config_->target_map().end()) { + continue; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] child=%s state=%s weight=%d picker=%p", + this, child_name.c_str(), + ConnectivityStateName(child->connectivity_state()), + child->weight(), child->picker_wrapper().get()); + } + switch (child->connectivity_state()) { + case GRPC_CHANNEL_READY: { + end += child->weight(); + picker_list.push_back(std::make_pair(end, child->picker_wrapper())); + break; + } + case GRPC_CHANNEL_CONNECTING: { + ++num_connecting; + break; + } + case GRPC_CHANNEL_IDLE: { + ++num_idle; + break; + } + case GRPC_CHANNEL_TRANSIENT_FAILURE: { + ++num_transient_failures; + break; + } + default: + GPR_UNREACHABLE_CODE(return ); + } + } + // Determine aggregated connectivity state. + grpc_connectivity_state connectivity_state; + if (!picker_list.empty()) { + connectivity_state = GRPC_CHANNEL_READY; + } else if (num_connecting > 0) { + connectivity_state = GRPC_CHANNEL_CONNECTING; + } else if (num_idle > 0) { + connectivity_state = GRPC_CHANNEL_IDLE; + } else { + connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] connectivity changed to %s", + this, ConnectivityStateName(connectivity_state)); + } + std::unique_ptr picker; + switch (connectivity_state) { + case GRPC_CHANNEL_READY: + picker = absl::make_unique(std::move(picker_list)); + break; + case GRPC_CHANNEL_CONNECTING: + case GRPC_CHANNEL_IDLE: + picker = + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker")); + break; + default: + picker = absl::make_unique( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "weighted_target: all children report state TRANSIENT_FAILURE")); + } + channel_control_helper()->UpdateState(connectivity_state, std::move(picker)); +} + +// +// WeightedTargetLb::WeightedChild +// + +WeightedTargetLb::WeightedChild::WeightedChild( + RefCountedPtr weighted_target_policy, + const std::string& name) + : weighted_target_policy_(std::move(weighted_target_policy)), name_(name) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] created WeightedChild %p for %s", + weighted_target_policy_.get(), this, name_.c_str()); + } +} + +WeightedTargetLb::WeightedChild::~WeightedChild() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: destroying child", + weighted_target_policy_.get(), this, name_.c_str()); + } + weighted_target_policy_.reset(DEBUG_LOCATION, "WeightedChild"); +} + +void WeightedTargetLb::WeightedChild::Orphan() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: shutting down child", + weighted_target_policy_.get(), this, name_.c_str()); + } + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + grpc_pollset_set_del_pollset_set( + child_policy_->interested_parties(), + weighted_target_policy_->interested_parties()); + child_policy_.reset(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_wrapper_.reset(); + if (delayed_removal_timer_callback_pending_) { + delayed_removal_timer_callback_pending_ = false; + grpc_timer_cancel(&delayed_removal_timer_); + } + shutdown_ = true; + Unref(); +} + +OrphanablePtr +WeightedTargetLb::WeightedChild::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = weighted_target_policy_->combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_weighted_target_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: Created new child " + "policy handler %p", + weighted_target_policy_.get(), this, name_.c_str(), + lb_policy.get()); + } + // Add the xDS's interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // xDS LB, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set( + lb_policy->interested_parties(), + weighted_target_policy_->interested_parties()); + return lb_policy; +} + +void WeightedTargetLb::WeightedChild::UpdateLocked( + const WeightedTargetLbConfig::ChildConfig& config, + ServerAddressList addresses, const grpc_channel_args* args) { + if (weighted_target_policy_->shutting_down_) return; + // Update child weight. + weight_ = config.weight; + // Reactivate if needed. + if (delayed_removal_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: reactivating", + weighted_target_policy_.get(), this, name_.c_str()); + } + delayed_removal_timer_callback_pending_ = false; + grpc_timer_cancel(&delayed_removal_timer_); + } + // Create child policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(args); + } + // Construct update args. + UpdateArgs update_args; + update_args.config = config.config; + update_args.addresses = std::move(addresses); + update_args.args = grpc_channel_args_copy(args); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: Updating child " + "policy handler %p", + weighted_target_policy_.get(), this, name_.c_str(), + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +void WeightedTargetLb::WeightedChild::ResetBackoffLocked() { + child_policy_->ResetBackoffLocked(); +} + +void WeightedTargetLb::WeightedChild::OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, std::unique_ptr picker) { + // Cache the picker in the WeightedChild. + picker_wrapper_ = MakeRefCounted(std::move(picker)); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: connectivity " + "state update: state=%s picker_wrapper=%p", + weighted_target_policy_.get(), this, name_.c_str(), + ConnectivityStateName(state), picker_wrapper_.get()); + } + // If the child reports IDLE, immediately tell it to exit idle. + if (state == GRPC_CHANNEL_IDLE) child_policy_->ExitIdleLocked(); + // Decide what state to report for aggregation purposes. + // If we haven't seen a failure since the last time we were in state + // READY, then we report the state change as-is. However, once we do see + // a failure, we report TRANSIENT_FAILURE and ignore any subsequent state + // changes until we go back into state READY. + if (!seen_failure_since_ready_) { + if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + seen_failure_since_ready_ = true; + } + } else { + if (state != GRPC_CHANNEL_READY) return; + seen_failure_since_ready_ = false; + } + connectivity_state_ = state; + // Notify the LB policy. + weighted_target_policy_->UpdateStateLocked(); +} + +void WeightedTargetLb::WeightedChild::DeactivateLocked() { + // If already deactivated, don't do that again. + if (weight_ == 0) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: deactivating", + weighted_target_policy_.get(), this, name_.c_str()); + } + // Set the child weight to 0 so that future picker won't contain this child. + weight_ = 0; + // Start a timer to delete the child. + Ref(DEBUG_LOCATION, "WeightedChild+timer").release(); + GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, + grpc_schedule_on_exec_ctx); + delayed_removal_timer_callback_pending_ = true; + grpc_timer_init(&delayed_removal_timer_, + ExecCtx::Get()->Now() + kChildRetentionIntervalMs, + &on_delayed_removal_timer_); +} + +void WeightedTargetLb::WeightedChild::OnDelayedRemovalTimer(void* arg, + grpc_error* error) { + WeightedChild* self = static_cast(arg); + self->weighted_target_policy_->combiner()->Run( + GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, + OnDelayedRemovalTimerLocked, self, nullptr), + GRPC_ERROR_REF(error)); +} + +void WeightedTargetLb::WeightedChild::OnDelayedRemovalTimerLocked( + void* arg, grpc_error* error) { + WeightedChild* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && + self->delayed_removal_timer_callback_pending_ && !self->shutdown_ && + self->weight_ == 0) { + self->delayed_removal_timer_callback_pending_ = false; + self->weighted_target_policy_->targets_.erase(self->name_); + } + self->Unref(DEBUG_LOCATION, "WeightedChild+timer"); +} + +// +// WeightedTargetLb::WeightedChild::Helper +// + +RefCountedPtr +WeightedTargetLb::WeightedChild::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return nullptr; + return weighted_child_->weighted_target_policy_->channel_control_helper() + ->CreateSubchannel(args); +} + +void WeightedTargetLb::WeightedChild::Helper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->OnConnectivityStateUpdateLocked(state, std::move(picker)); +} + +void WeightedTargetLb::WeightedChild::Helper::RequestReresolution() { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->weighted_target_policy_->channel_control_helper() + ->RequestReresolution(); +} + +void WeightedTargetLb::WeightedChild::Helper::AddTraceEvent( + TraceSeverity severity, StringView message) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->weighted_target_policy_->channel_control_helper() + ->AddTraceEvent(severity, message); +} + +// +// factory +// + +class WeightedTargetLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args)); + } + + const char* name() const override { return kWeightedTarget; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // weighted_target was mentioned as a policy in the deprecated + // loadBalancingPolicy field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:weighted_target policy requires " + "configuration. Please use loadBalancingConfig field of service " + "config instead."); + return nullptr; + } + std::vector error_list; + // Weight map. + WeightedTargetLbConfig::TargetMap target_map; + auto it = json.object_value().find("targets"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:targets error:required field not present")); + } else if (it->second.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:targets error:type should be object")); + } else { + for (const auto& p : it->second.object_value()) { + WeightedTargetLbConfig::ChildConfig child_config; + std::vector child_errors = + ParseChildConfig(p.second, &child_config); + if (!child_errors.empty()) { + // Can't use GRPC_ERROR_CREATE_FROM_VECTOR() here, because the error + // string is not static in this case. + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:targets key:", p.first).c_str()); + for (grpc_error* child_error : child_errors) { + error = grpc_error_add_child(error, child_error); + } + error_list.push_back(error); + } else { + target_map[p.first] = std::move(child_config); + } + } + } + if (!error_list.empty()) { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "weighted_target_experimental LB policy config", &error_list); + return nullptr; + } + return MakeRefCounted(std::move(target_map)); + } + + private: + static std::vector ParseChildConfig( + const Json& json, WeightedTargetLbConfig::ChildConfig* child_config) { + std::vector error_list; + if (json.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "value should be of type object")); + return error_list; + } + // Weight. + auto it = json.object_value().find("weight"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "required field \"weight\" not specified")); + } else if (it->second.type() != Json::Type::NUMBER) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:must be of type number")); + } else { + child_config->weight = + gpr_parse_nonnegative_int(it->second.string_value().c_str()); + if (child_config->weight == -1) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:unparseable value")); + } else if (child_config->weight == 0) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:value must be greater than zero")); + } + } + // Child policy. + it = json.object_value().find("childPolicy"); + if (it != json.object_value().end()) { + grpc_error* parse_error = GRPC_ERROR_NONE; + child_config->config = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(it->second, + &parse_error); + if (child_config->config == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + std::vector child_errors; + child_errors.push_back(parse_error); + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); + } + } + return error_list; + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_weighted_target_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_weighted_target_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc index 51e250a3cad..29f571de414 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc @@ -37,9 +37,9 @@ namespace { constexpr char kCds[] = "cds_experimental"; // Config for this LB policy. -class CdsConfig : public LoadBalancingPolicy::Config { +class CdsLbConfig : public LoadBalancingPolicy::Config { public: - explicit CdsConfig(std::string cluster) : cluster_(std::move(cluster)) {} + explicit CdsLbConfig(std::string cluster) : cluster_(std::move(cluster)) {} const std::string& cluster() const { return cluster_; } const char* name() const override { return kCds; } @@ -50,7 +50,7 @@ class CdsConfig : public LoadBalancingPolicy::Config { // CDS LB policy. class CdsLb : public LoadBalancingPolicy { public: - explicit CdsLb(Args args); + CdsLb(RefCountedPtr xds_client, Args args); const char* name() const override { return kCds; } @@ -89,7 +89,7 @@ class CdsLb : public LoadBalancingPolicy { void ShutdownLocked() override; - RefCountedPtr config_; + RefCountedPtr config_; // Current channel args from the resolver. const grpc_channel_args* args_ = nullptr; @@ -124,21 +124,37 @@ void CdsLb::ClusterWatcher::OnClusterChanged(XdsApi::CdsUpdate cluster_data) { } // Construct config for child policy. Json::Object child_config = { - {"edsServiceName", - (cluster_data.eds_service_name.empty() ? parent_->config_->cluster() - : cluster_data.eds_service_name)}, + {"clusterName", parent_->config_->cluster()}, + {"localityPickingPolicy", + Json::Array{ + Json::Object{ + {"weighted_target_experimental", + Json::Object{ + {"targets", Json::Object()}, + }}, + }, + }}, + {"endpointPickingPolicy", + Json::Array{ + Json::Object{ + {"round_robin", Json::Object()}, + }, + }}, }; + if (!cluster_data.eds_service_name.empty()) { + child_config["edsServiceName"] = cluster_data.eds_service_name; + } if (cluster_data.lrs_load_reporting_server_name.has_value()) { child_config["lrsLoadReportingServerName"] = cluster_data.lrs_load_reporting_server_name.value(); } Json json = Json::Array{ Json::Object{ - {"xds_experimental", std::move(child_config)}, + {"eds_experimental", std::move(child_config)}, }, }; if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - std::string json_str = json.Dump(); + std::string json_str = json.Dump(/*indent=*/1); gpr_log(GPR_INFO, "[cdslb %p] generated config for child policy: %s", parent_.get(), json_str.c_str()); } @@ -156,19 +172,19 @@ void CdsLb::ClusterWatcher::OnClusterChanged(XdsApi::CdsUpdate cluster_data) { args.args = parent_->args_; args.channel_control_helper = absl::make_unique(parent_->Ref()); parent_->child_policy_ = - LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy( - "xds_experimental", std::move(args)); + LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(config->name(), + std::move(args)); if (parent_->child_policy_ == nullptr) { OnError(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "failed to create xds_experimental child policy")); + "failed to create child policy")); return; } grpc_pollset_set_add_pollset_set( parent_->child_policy_->interested_parties(), parent_->interested_parties()); if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] created child policy xds_experimental (%p)", - parent_.get(), parent_->child_policy_.get()); + gpr_log(GPR_INFO, "[cdslb %p] created child policy %s (%p)", + parent_.get(), config->name(), parent_->child_policy_.get()); } } // Update child policy. @@ -232,9 +248,8 @@ void CdsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { // CdsLb // -CdsLb::CdsLb(Args args) - : LoadBalancingPolicy(std::move(args)), - xds_client_(XdsClient::GetFromChannelArgs(*args.args)) { +CdsLb::CdsLb(RefCountedPtr xds_client, Args args) + : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { gpr_log(GPR_INFO, "[cdslb %p] created -- using xds client %p from channel", this, xds_client_.get()); @@ -313,11 +328,19 @@ void CdsLb::UpdateLocked(UpdateArgs args) { // factory // -class CdsFactory : public LoadBalancingPolicyFactory { +class CdsLbFactory : public LoadBalancingPolicyFactory { public: OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args)); + RefCountedPtr xds_client = + XdsClient::GetFromChannelArgs(*args.args); + if (xds_client == nullptr) { + gpr_log(GPR_ERROR, + "XdsClient not present in channel args -- cannot instantiate " + "cds LB policy"); + return nullptr; + } + return MakeOrphanable(std::move(xds_client), std::move(args)); } const char* name() const override { return kCds; } @@ -349,7 +372,7 @@ class CdsFactory : public LoadBalancingPolicyFactory { *error = GRPC_ERROR_CREATE_FROM_VECTOR("Cds Parser", &error_list); return nullptr; } - return MakeRefCounted(std::move(cluster)); + return MakeRefCounted(std::move(cluster)); } }; @@ -364,7 +387,7 @@ class CdsFactory : public LoadBalancingPolicyFactory { void grpc_lb_policy_cds_init() { grpc_core::LoadBalancingPolicyRegistry::Builder:: RegisterLoadBalancingPolicyFactory( - absl::make_unique()); + absl::make_unique()); } void grpc_lb_policy_cds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc new file mode 100644 index 00000000000..2f1acdb9f80 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc @@ -0,0 +1,1172 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" + +#include + +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/ext/filters/client_channel/server_address.h" +#include "src/core/ext/filters/client_channel/xds/xds_client.h" +#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/uri/uri_parser.h" + +#define GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT 10000 + +namespace grpc_core { + +TraceFlag grpc_lb_eds_trace(false, "eds_lb"); + +namespace { + +constexpr char kXds[] = "xds_experimental"; +constexpr char kEds[] = "eds_experimental"; + +// Config for EDS LB policy. +class EdsLbConfig : public LoadBalancingPolicy::Config { + public: + EdsLbConfig(const char* name, std::string cluster_name, + std::string eds_service_name, + absl::optional lrs_load_reporting_server_name, + Json locality_picking_policy, Json endpoint_picking_policy, + RefCountedPtr fallback_policy) + : name_(name), + cluster_name_(std::move(cluster_name)), + eds_service_name_(std::move(eds_service_name)), + lrs_load_reporting_server_name_( + std::move(lrs_load_reporting_server_name)), + locality_picking_policy_(std::move(locality_picking_policy)), + endpoint_picking_policy_(std::move(endpoint_picking_policy)), + fallback_policy_(std::move(fallback_policy)) {} + + const char* name() const override { return name_; } + + const std::string& cluster_name() const { return cluster_name_; } + const std::string& eds_service_name() const { return eds_service_name_; } + const absl::optional& lrs_load_reporting_server_name() const { + return lrs_load_reporting_server_name_; + }; + const Json& locality_picking_policy() const { + return locality_picking_policy_; + } + const Json& endpoint_picking_policy() const { + return endpoint_picking_policy_; + } + RefCountedPtr fallback_policy() const { + return fallback_policy_; + } + + private: + const char* name_; + std::string cluster_name_; + std::string eds_service_name_; + absl::optional lrs_load_reporting_server_name_; + Json locality_picking_policy_; + Json endpoint_picking_policy_; + RefCountedPtr fallback_policy_; +}; + +// EDS LB policy. +class EdsLb : public LoadBalancingPolicy { + public: + EdsLb(const char* name, Args args); + + const char* name() const override { return name_; } + + void UpdateLocked(UpdateArgs args) override; + void ResetBackoffLocked() override; + + private: + class EndpointWatcher; + + // A simple wrapper for ref-counting a picker from the child policy. + class ChildPickerWrapper : public RefCounted { + public: + explicit ChildPickerWrapper(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A picker that handles drops. + class DropPicker : public SubchannelPicker { + public: + explicit DropPicker(EdsLb* eds_policy); + + PickResult Pick(PickArgs args) override; + + private: + RefCountedPtr drop_config_; + RefCountedPtr drop_stats_; + RefCountedPtr child_picker_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr eds_policy) + : eds_policy_(std::move(eds_policy)) {} + + ~Helper() { eds_policy_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + // This is a no-op, because we get the addresses from the xds + // client, which is a watch-based API. + void RequestReresolution() override {} + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr eds_policy_; + }; + + class FallbackHelper : public ChannelControlHelper { + public: + explicit FallbackHelper(RefCountedPtr parent) + : parent_(std::move(parent)) {} + + ~FallbackHelper() { parent_.reset(DEBUG_LOCATION, "FallbackHelper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr parent_; + }; + + ~EdsLb(); + + void ShutdownLocked() override; + + void UpdatePriorityList(XdsApi::PriorityListUpdate priority_list_update); + void UpdateChildPolicyLocked(); + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + ServerAddressList CreateChildPolicyAddressesLocked(); + RefCountedPtr CreateChildPolicyConfigLocked(); + grpc_channel_args* CreateChildPolicyArgsLocked( + const grpc_channel_args* args_in); + void MaybeUpdateDropPickerLocked(); + + // Methods for dealing with fallback state. + void MaybeCancelFallbackAtStartupChecks(); + static void OnFallbackTimer(void* arg, grpc_error* error); + static void OnFallbackTimerLocked(void* arg, grpc_error* error); + void UpdateFallbackPolicyLocked(); + OrphanablePtr CreateFallbackPolicyLocked( + const grpc_channel_args* args); + void MaybeExitFallbackMode(); + + // Caller must ensure that config_ is set before calling. + const StringView GetEdsResourceName() const { + if (xds_client_from_channel_ == nullptr) return server_name_; + if (!config_->eds_service_name().empty()) { + return config_->eds_service_name(); + } + return config_->cluster_name(); + } + + // Returns a pair containing the cluster and eds_service_name to use + // for LRS load reporting. + // Caller must ensure that config_ is set before calling. + std::pair GetLrsClusterKey() const { + if (xds_client_from_channel_ == nullptr) return {server_name_, nullptr}; + return {config_->cluster_name(), config_->eds_service_name()}; + } + + XdsClient* xds_client() const { + return xds_client_from_channel_ != nullptr ? xds_client_from_channel_.get() + : xds_client_.get(); + } + + // Policy name (kXds or kEds). + const char* name_; + + // Server name from target URI. + std::string server_name_; + + // Current channel args and config from the resolver. + const grpc_channel_args* args_ = nullptr; + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // The xds client and endpoint watcher. + // If we get the XdsClient from the channel, we store it in + // xds_client_from_channel_; if we create it ourselves, we store it in + // xds_client_. + RefCountedPtr xds_client_from_channel_; + OrphanablePtr xds_client_; + // A pointer to the endpoint watcher, to be used when cancelling the watch. + // Note that this is not owned, so this pointer must never be derefernced. + EndpointWatcher* endpoint_watcher_ = nullptr; + // The latest data from the endpoint watcher. + XdsApi::PriorityListUpdate priority_list_update_; + // State used to retain child policy names for priority policy. + std::vector priority_child_numbers_; + + RefCountedPtr drop_config_; + RefCountedPtr drop_stats_; + + OrphanablePtr child_policy_; + + // The latest state and picker returned from the child policy. + grpc_connectivity_state child_state_; + RefCountedPtr child_picker_; + + // Non-null iff we are in fallback mode. + OrphanablePtr fallback_policy_; + + // Whether the checks for fallback at startup are ALL pending. There are + // several cases where this can be reset: + // 1. The fallback timer fires, we enter fallback mode. + // 2. Before the fallback timer fires, the endpoint watcher reports an + // error, we enter fallback mode. + // 3. Before the fallback timer fires, if any child policy in the locality map + // becomes READY, we cancel the fallback timer. + bool fallback_at_startup_checks_pending_ = false; + // Timeout in milliseconds for before using fallback backend addresses. + // 0 means not using fallback. + const grpc_millis lb_fallback_timeout_ms_; + // The backend addresses from the resolver. + ServerAddressList fallback_backend_addresses_; + // Fallback timer. + grpc_timer lb_fallback_timer_; + grpc_closure lb_on_fallback_; +}; + +// +// EdsLb::DropPicker +// + +EdsLb::DropPicker::DropPicker(EdsLb* eds_policy) + : drop_config_(eds_policy->drop_config_), + drop_stats_(eds_policy->drop_stats_), + child_picker_(eds_policy->child_picker_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] constructed new drop picker %p", eds_policy, + this); + } +} + +EdsLb::PickResult EdsLb::DropPicker::Pick(PickArgs args) { + // Handle drop. + const std::string* drop_category; + if (drop_config_->ShouldDrop(&drop_category)) { + if (drop_stats_ != nullptr) drop_stats_->AddCallDropped(*drop_category); + PickResult result; + result.type = PickResult::PICK_COMPLETE; + return result; + } + // If we're not dropping all calls, we should always have a child picker. + if (child_picker_ == nullptr) { // Should never happen. + PickResult result; + result.type = PickResult::PICK_FAILED; + result.error = + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "eds drop picker not given any child picker"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); + return result; + } + // Not dropping, so delegate to child's picker. + return child_picker_->Pick(args); +} + +// +// EdsLb::Helper +// + +RefCountedPtr EdsLb::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (eds_policy_->shutting_down_) return nullptr; + return eds_policy_->channel_control_helper()->CreateSubchannel(args); +} + +void EdsLb::Helper::UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) { + if (eds_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] child policy updated state=%s picker=%p", + eds_policy_.get(), ConnectivityStateName(state), picker.get()); + } + // Save the state and picker. + eds_policy_->child_state_ = state; + eds_policy_->child_picker_ = + MakeRefCounted(std::move(picker)); + // If the new state is READY, cancel the fallback-at-startup checks. + if (state == GRPC_CHANNEL_READY) { + eds_policy_->MaybeCancelFallbackAtStartupChecks(); + eds_policy_->MaybeExitFallbackMode(); + } + // TODO(roth): If the child reports TRANSIENT_FAILURE and the + // fallback-at-startup checks are pending, we should probably go into + // fallback mode immediately (cancelling the fallback-at-startup timer + // if needed). + // Wrap the picker in a DropPicker and pass it up. + eds_policy_->MaybeUpdateDropPickerLocked(); +} + +void EdsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { + if (eds_policy_->shutting_down_) return; + eds_policy_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// EdsLb::FallbackHelper +// + +RefCountedPtr EdsLb::FallbackHelper::CreateSubchannel( + const grpc_channel_args& args) { + if (parent_->shutting_down_) return nullptr; + return parent_->channel_control_helper()->CreateSubchannel(args); +} + +void EdsLb::FallbackHelper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->UpdateState(state, std::move(picker)); +} + +void EdsLb::FallbackHelper::RequestReresolution() { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->RequestReresolution(); +} + +void EdsLb::FallbackHelper::AddTraceEvent(TraceSeverity severity, + StringView message) { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// EdsLb::EndpointWatcher +// + +class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { + public: + explicit EndpointWatcher(RefCountedPtr eds_policy) + : eds_policy_(std::move(eds_policy)) {} + + ~EndpointWatcher() { eds_policy_.reset(DEBUG_LOCATION, "EndpointWatcher"); } + + void OnEndpointChanged(XdsApi::EdsUpdate update) override { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Received EDS update from xds client", + eds_policy_.get()); + } + // If the balancer tells us to drop all the calls, we should exit fallback + // mode immediately. + if (update.drop_config->drop_all()) eds_policy_->MaybeExitFallbackMode(); + // Update the drop config. + const bool drop_config_changed = + eds_policy_->drop_config_ == nullptr || + *eds_policy_->drop_config_ != *update.drop_config; + if (drop_config_changed) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating drop config", eds_policy_.get()); + } + eds_policy_->drop_config_ = std::move(update.drop_config); + eds_policy_->MaybeUpdateDropPickerLocked(); + } + // Update priority and locality info. + if (eds_policy_->priority_list_update_ == update.priority_list_update) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, + "[edslb %p] Incoming locality update identical to current, " + "ignoring. (drop_config_changed=%d)", + eds_policy_.get(), drop_config_changed); + } + return; + } + // Update the child policy with the new priority and endpoint data. + eds_policy_->UpdatePriorityList(std::move(update.priority_list_update)); + } + + void OnError(grpc_error* error) override { + // If the fallback-at-startup checks are pending, go into fallback mode + // immediately. This short-circuits the timeout for the + // fallback-at-startup case. + if (eds_policy_->fallback_at_startup_checks_pending_) { + gpr_log(GPR_ERROR, + "[edslb %p] xds watcher reported error; entering fallback " + "mode: %s", + eds_policy_.get(), grpc_error_string(error)); + eds_policy_->fallback_at_startup_checks_pending_ = false; + grpc_timer_cancel(&eds_policy_->lb_fallback_timer_); + eds_policy_->UpdateFallbackPolicyLocked(); + // If the xds call failed, request re-resolution. + // TODO(roth): We check the error string contents here to + // differentiate between the xds call failing and the xds channel + // going into TRANSIENT_FAILURE. This is a pretty ugly hack, + // but it's okay for now, since we're not yet sure whether we will + // continue to support the current fallback functionality. If we + // decide to keep the fallback approach, then we should either + // find a cleaner way to expose the difference between these two + // cases or decide that we're okay re-resolving in both cases. + // Note that even if we do keep the current fallback functionality, + // this re-resolution will only be necessary if we are going to be + // using this LB policy with resolvers other than the xds resolver. + if (strstr(grpc_error_string(error), "xds call failed")) { + eds_policy_->channel_control_helper()->RequestReresolution(); + } + } + GRPC_ERROR_UNREF(error); + } + + private: + RefCountedPtr eds_policy_; +}; + +// +// EdsLb public methods +// + +EdsLb::EdsLb(const char* name, Args args) + : LoadBalancingPolicy(std::move(args)), + name_(name), + xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)), + lb_fallback_timeout_ms_(grpc_channel_args_find_integer( + args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, + {GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT, 0, INT_MAX})) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] created -- xds client from channel: %p", this, + xds_client_from_channel_.get()); + } + // Record server name. + const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI); + const char* server_uri = grpc_channel_arg_get_string(arg); + GPR_ASSERT(server_uri != nullptr); + grpc_uri* uri = grpc_uri_parse(server_uri, true); + GPR_ASSERT(uri->path[0] != '\0'); + server_name_ = uri->path[0] == '/' ? uri->path + 1 : uri->path; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] server name from channel: %s", this, + server_name_.c_str()); + } + grpc_uri_destroy(uri); +} + +EdsLb::~EdsLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] destroying xds LB policy", this); + } + grpc_channel_args_destroy(args_); +} + +void EdsLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] shutting down", this); + } + shutting_down_ = true; + MaybeCancelFallbackAtStartupChecks(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + child_picker_.reset(); + if (child_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + interested_parties()); + child_policy_.reset(); + } + if (fallback_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(), + interested_parties()); + fallback_policy_.reset(); + } + drop_stats_.reset(); + // Cancel the endpoint watch here instead of in our dtor if we are using the + // xds resolver, because the watcher holds a ref to us and we might not be + // destroying the XdsClient, leading to a situation where this LB policy is + // never destroyed. + if (xds_client_from_channel_ != nullptr) { + if (config_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] cancelling xds watch for %s", this, + std::string(GetEdsResourceName()).c_str()); + } + xds_client()->CancelEndpointDataWatch(GetEdsResourceName(), + endpoint_watcher_); + } + xds_client_from_channel_.reset(); + } + xds_client_.reset(); +} + +void EdsLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Received update", this); + } + const bool is_initial_update = args_ == nullptr; + // Update config. + auto old_config = std::move(config_); + config_ = std::move(args.config); + // Update fallback address list. + fallback_backend_addresses_ = std::move(args.addresses); + // Update args. + grpc_channel_args_destroy(args_); + args_ = args.args; + args.args = nullptr; + // Update the existing fallback policy. The fallback policy config and/or the + // fallback addresses may be new. + if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked(); + if (is_initial_update) { + // Initialize XdsClient. + if (xds_client_from_channel_ == nullptr) { + grpc_error* error = GRPC_ERROR_NONE; + xds_client_ = MakeOrphanable( + combiner(), interested_parties(), GetEdsResourceName(), + nullptr /* service config watcher */, *args_, &error); + // TODO(roth): If we decide that we care about fallback mode, add + // proper error handling here. + GPR_ASSERT(error == GRPC_ERROR_NONE); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Created xds client %p", this, + xds_client_.get()); + } + } + // Start fallback-at-startup checks. + grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_; + Ref(DEBUG_LOCATION, "on_fallback_timer").release(); // Held by closure + GRPC_CLOSURE_INIT(&lb_on_fallback_, &EdsLb::OnFallbackTimer, this, + grpc_schedule_on_exec_ctx); + fallback_at_startup_checks_pending_ = true; + grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_); + } + // Update drop stats for load reporting if needed. + if (is_initial_update || config_->lrs_load_reporting_server_name() != + old_config->lrs_load_reporting_server_name()) { + drop_stats_.reset(); + if (config_->lrs_load_reporting_server_name().has_value()) { + const auto key = GetLrsClusterKey(); + drop_stats_ = xds_client()->AddClusterDropStats( + config_->lrs_load_reporting_server_name().value(), + key.first /*cluster_name*/, key.second /*eds_service_name*/); + } + MaybeUpdateDropPickerLocked(); + } + // Update child policy if needed. + // Note that this comes after updating drop_stats_, since we want that + // to be used by any new picker we create here. + if (child_policy_ != nullptr) UpdateChildPolicyLocked(); + // Create endpoint watcher if needed. + if (is_initial_update) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] starting xds watch for %s", this, + std::string(GetEdsResourceName()).c_str()); + } + auto watcher = absl::make_unique( + Ref(DEBUG_LOCATION, "EndpointWatcher")); + endpoint_watcher_ = watcher.get(); + xds_client()->WatchEndpointData(GetEdsResourceName(), std::move(watcher)); + } +} + +void EdsLb::ResetBackoffLocked() { + // When the XdsClient is instantiated in the resolver instead of in this + // LB policy, this is done via the resolver, so we don't need to do it + // for xds_client_from_channel_ here. + if (xds_client_ != nullptr) xds_client_->ResetBackoff(); + if (child_policy_ != nullptr) { + child_policy_->ResetBackoffLocked(); + } + if (fallback_policy_ != nullptr) { + fallback_policy_->ResetBackoffLocked(); + } +} + +// +// child policy-related methods +// + +void EdsLb::UpdatePriorityList( + XdsApi::PriorityListUpdate priority_list_update) { + // Build some maps from locality to child number and the reverse from + // the old data in priority_list_update_ and priority_child_numbers_. + std::map + locality_child_map; + std::map> child_locality_map; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + size_t child_number = priority_child_numbers_[priority]; + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + locality_child_map[locality_name] = child_number; + child_locality_map[child_number].insert(locality_name); + } + } + // Construct new list of children. + std::vector priority_child_numbers; + for (uint32_t priority = 0; priority < priority_list_update.size(); + ++priority) { + auto* locality_map = priority_list_update.Find(priority); + GPR_ASSERT(locality_map != nullptr); + absl::optional child_number; + // If one of the localities in this priority already existed, reuse its + // child number. + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + if (!child_number.has_value()) { + auto it = locality_child_map.find(locality_name); + if (it != locality_child_map.end()) { + child_number = it->second; + locality_child_map.erase(it); + // Remove localities that *used* to be in this child number, so + // that we don't incorrectly reuse this child number for a + // subsequent priority. + for (XdsLocalityName* old_locality : + child_locality_map[*child_number]) { + locality_child_map.erase(old_locality); + } + } + } else { + // Remove all localities that are now in this child number, so + // that we don't accidentally reuse this child number for a + // subsequent priority. + locality_child_map.erase(locality_name); + } + } + // If we didn't find an existing child number, assign a new one. + if (!child_number.has_value()) { + for (child_number = 0; + child_locality_map.find(*child_number) != child_locality_map.end(); + ++(*child_number)) + ; + // Add entry so we know that the child number is in use. + // (Don't need to add the list of localities, since we won't use them.) + child_locality_map[*child_number]; + } + priority_child_numbers.push_back(*child_number); + } + // Save update. + priority_list_update_ = std::move(priority_list_update); + priority_child_numbers_ = std::move(priority_child_numbers); + // Update child policy. + UpdateChildPolicyLocked(); +} + +ServerAddressList EdsLb::CreateChildPolicyAddressesLocked() { + ServerAddressList addresses; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + std::string priority_child_name = + absl::StrCat("child", priority_child_numbers_[priority]); + const auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + for (const auto& p : locality_map->localities) { + const auto& locality_name = p.first; + const auto& locality = p.second; + std::vector hierarchical_path = { + priority_child_name, locality_name->AsHumanReadableString()}; + for (size_t i = 0; i < locality.serverlist.size(); ++i) { + const ServerAddress& address = locality.serverlist[i]; + grpc_arg new_arg = MakeHierarchicalPathArg(hierarchical_path); + grpc_channel_args* args = + grpc_channel_args_copy_and_add(address.args(), &new_arg, 1); + addresses.emplace_back(address.address(), args); + } + } + } + return addresses; +} + +RefCountedPtr +EdsLb::CreateChildPolicyConfigLocked() { + Json::Object priority_children; + Json::Array priority_priorities; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + const auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + Json::Object weighted_targets; + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + const auto& locality = p.second; + // Construct JSON object containing locality name. + Json::Object locality_name_json; + if (!locality_name->region().empty()) { + locality_name_json["region"] = locality_name->region(); + } + if (!locality_name->zone().empty()) { + locality_name_json["zone"] = locality_name->zone(); + } + if (!locality_name->sub_zone().empty()) { + locality_name_json["subzone"] = locality_name->sub_zone(); + } + // Construct endpoint-picking policy. + // Wrap it in the LRS policy if load reporting is enabled. + Json endpoint_picking_policy; + if (config_->lrs_load_reporting_server_name().has_value()) { + const auto key = GetLrsClusterKey(); + Json::Object lrs_config = { + {"clusterName", std::string(key.first)}, + {"locality", std::move(locality_name_json)}, + {"lrsLoadReportingServerName", + config_->lrs_load_reporting_server_name().value()}, + {"childPolicy", config_->endpoint_picking_policy()}, + }; + if (!key.second.empty()) { + lrs_config["edsServiceName"] = std::string(key.second); + } + endpoint_picking_policy = Json::Array{Json::Object{ + {"lrs_experimental", std::move(lrs_config)}, + }}; + } else { + endpoint_picking_policy = config_->endpoint_picking_policy(); + } + // Add weighted target entry. + weighted_targets[locality_name->AsHumanReadableString()] = Json::Object{ + {"weight", locality.lb_weight}, + {"childPolicy", std::move(endpoint_picking_policy)}, + }; + } + // Add priority entry. + const size_t child_number = priority_child_numbers_[priority]; + std::string child_name = absl::StrCat("child", child_number); + priority_priorities.emplace_back(child_name); + Json locality_picking_config = config_->locality_picking_policy(); + Json::Object& config = + *(*locality_picking_config.mutable_array())[0].mutable_object(); + auto it = config.begin(); + GPR_ASSERT(it != config.end()); + (*it->second.mutable_object())["targets"] = std::move(weighted_targets); + priority_children[child_name] = Json::Object{ + {"config", std::move(locality_picking_config)}, + }; + } + Json json = Json::Array{Json::Object{ + {"priority_experimental", + Json::Object{ + {"children", std::move(priority_children)}, + {"priorities", std::move(priority_priorities)}, + }}, + }}; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + std::string json_str = json.Dump(/*indent=*/1); + gpr_log(GPR_INFO, "[edslb %p] generated config for child policy: %s", this, + json_str.c_str()); + } + grpc_error* error = GRPC_ERROR_NONE; + RefCountedPtr config = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(json, &error); + if (error != GRPC_ERROR_NONE) { + // This should never happen, but if it does, we basically have no + // way to fix it, so we put the channel in TRANSIENT_FAILURE. + gpr_log(GPR_ERROR, + "[edslb %p] error parsing generated child policy config -- " + "will put channel in TRANSIENT_FAILURE: %s", + this, grpc_error_string(error)); + error = grpc_error_set_int( + grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "eds LB policy: error parsing generated child policy config"), + error), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, + absl::make_unique(error)); + return nullptr; + } + return config; +} + +void EdsLb::UpdateChildPolicyLocked() { + if (shutting_down_) return; + UpdateArgs update_args; + update_args.config = CreateChildPolicyConfigLocked(); + if (update_args.config == nullptr) return; + update_args.addresses = CreateChildPolicyAddressesLocked(); + update_args.args = CreateChildPolicyArgsLocked(args_); + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(update_args.args); + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating child policy %p", this, + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +grpc_channel_args* EdsLb::CreateChildPolicyArgsLocked( + const grpc_channel_args* args) { + absl::InlinedVector args_to_add = { + // A channel arg indicating if the target is a backend inferred from an + // xds load balancer. + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER), + 1), + // Inhibit client-side health checking, since the balancer does + // this for us. + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1), + }; + if (xds_client_from_channel_ == nullptr) { + args_to_add.emplace_back(xds_client_->MakeChannelArg()); + } + return grpc_channel_args_copy_and_add(args, args_to_add.data(), + args_to_add.size()); +} + +OrphanablePtr EdsLb::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy( + "priority_experimental", std::move(lb_policy_args)); + if (GPR_UNLIKELY(lb_policy == nullptr)) { + gpr_log(GPR_ERROR, "[edslb %p] failure creating child policy", this); + return nullptr; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p]: Created new child policy %p", this, + lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void EdsLb::MaybeUpdateDropPickerLocked() { + // If we are in fallback mode, don't override the picker. + if (fallback_policy_ != nullptr) return; + // If we're dropping all calls, report READY, regardless of what (or + // whether) the child has reported. + if (drop_config_ != nullptr && drop_config_->drop_all()) { + channel_control_helper()->UpdateState(GRPC_CHANNEL_READY, + absl::make_unique(this)); + return; + } + // Update only if we have a child picker. + if (child_picker_ != nullptr) { + channel_control_helper()->UpdateState(child_state_, + absl::make_unique(this)); + } +} + +// +// fallback-related methods +// + +void EdsLb::MaybeCancelFallbackAtStartupChecks() { + if (!fallback_at_startup_checks_pending_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Cancelling fallback timer", this); + } + grpc_timer_cancel(&lb_fallback_timer_); + fallback_at_startup_checks_pending_ = false; +} + +void EdsLb::OnFallbackTimer(void* arg, grpc_error* error) { + EdsLb* edslb_policy = static_cast(arg); + edslb_policy->combiner()->Run( + GRPC_CLOSURE_INIT(&edslb_policy->lb_on_fallback_, + &EdsLb::OnFallbackTimerLocked, edslb_policy, nullptr), + GRPC_ERROR_REF(error)); +} + +void EdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) { + EdsLb* edslb_policy = static_cast(arg); + // If some fallback-at-startup check is done after the timer fires but before + // this callback actually runs, don't fall back. + if (edslb_policy->fallback_at_startup_checks_pending_ && + !edslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) { + gpr_log(GPR_INFO, + "[edslb %p] Child policy not ready after fallback timeout; " + "entering fallback mode", + edslb_policy); + edslb_policy->fallback_at_startup_checks_pending_ = false; + edslb_policy->UpdateFallbackPolicyLocked(); + } + edslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer"); +} + +void EdsLb::UpdateFallbackPolicyLocked() { + if (shutting_down_) return; + // Create policy if needed. + if (fallback_policy_ == nullptr) { + fallback_policy_ = CreateFallbackPolicyLocked(args_); + } + // Construct update args. + UpdateArgs update_args; + update_args.addresses = fallback_backend_addresses_; + update_args.config = config_->fallback_policy(); + update_args.args = grpc_channel_args_copy(args_); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating fallback child policy handler %p", + this, fallback_policy_.get()); + } + fallback_policy_->UpdateLocked(std::move(update_args)); +} + +OrphanablePtr EdsLb::CreateFallbackPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "FallbackHelper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_eds_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Created new fallback child policy handler %p", + this, lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void EdsLb::MaybeExitFallbackMode() { + if (fallback_policy_ == nullptr) return; + gpr_log(GPR_INFO, "[edslb %p] Exiting fallback mode", this); + fallback_policy_.reset(); +} + +// +// factory +// + +class EdsLbFactory : public LoadBalancingPolicyFactory { + public: + explicit EdsLbFactory(const char* name) : name_(name) {} + + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args), &grpc_lb_eds_trace, + name_); + } + + const char* name() const override { return name_; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // xds was mentioned as a policy in the deprecated loadBalancingPolicy + // field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:eds policy requires configuration. " + "Please use loadBalancingConfig field of service config instead."); + return nullptr; + } + std::vector error_list; + // EDS service name. + std::string eds_service_name; + auto it = json.object_value().find("edsServiceName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:edsServiceName error:type should be string")); + } else { + eds_service_name = it->second.string_value(); + } + } + // Cluster name. + std::string cluster_name; + if (name_ == kEds) { + it = json.object_value().find("clusterName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:type should be string")); + } else { + cluster_name = it->second.string_value(); + } + } else { + // For xds policy, this field does not exist in the config, so it + // will always be set to the same value as edsServiceName. + cluster_name = eds_service_name; + } + // LRS load reporting server name. + absl::optional lrs_load_reporting_server_name; + it = json.object_value().find("lrsLoadReportingServerName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:type should be string")); + } else { + lrs_load_reporting_server_name.emplace(it->second.string_value()); + } + } + // Locality-picking policy. Not supported for xds policy. + Json locality_picking_policy = Json::Array{ + Json::Object{ + {"weighted_target_experimental", + Json::Object{ + {"targets", Json::Object()}, + }}, + }, + }; + if (name_ == kEds) { + it = json.object_value().find("localityPickingPolicy"); + if (it != json.object_value().end()) { + locality_picking_policy = it->second; + } + } + grpc_error* parse_error = GRPC_ERROR_NONE; + if (LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + locality_picking_policy, &parse_error) == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "localityPickingPolicy", &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + // Endpoint-picking policy. Called "childPolicy" for xds policy. + const char* field_name = + name_ == kEds ? "endpointPickingPolicy" : "childPolicy"; + Json endpoint_picking_policy; + it = json.object_value().find(field_name); + if (it == json.object_value().end()) { + endpoint_picking_policy = Json::Array{ + Json::Object{ + {"round_robin", Json::Object()}, + }, + }; + } else { + endpoint_picking_policy = it->second; + } + parse_error = GRPC_ERROR_NONE; + if (LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + endpoint_picking_policy, &parse_error) == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + field_name, &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + // Fallback policy. + Json fallback_policy_config; + it = json.object_value().find("fallbackPolicy"); + if (it == json.object_value().end()) { + fallback_policy_config = Json::Array{Json::Object{ + {"round_robin", Json::Object()}, + }}; + } else { + fallback_policy_config = it->second; + } + parse_error = GRPC_ERROR_NONE; + RefCountedPtr fallback_policy = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + fallback_policy_config, &parse_error); + if (fallback_policy == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "fallbackPolicy", &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + error_list.push_back(parse_error); + } + if (error_list.empty()) { + return MakeRefCounted( + name_, std::move(cluster_name), std::move(eds_service_name), + std::move(lrs_load_reporting_server_name), + std::move(locality_picking_policy), + std::move(endpoint_picking_policy), std::move(fallback_policy)); + } else { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "eds_experimental LB policy config", &error_list); + return nullptr; + } + } + + private: + class EdsChildHandler : public ChildPolicyHandler { + public: + EdsChildHandler(Args args, TraceFlag* tracer, const char* name) + : ChildPolicyHandler(std::move(args), tracer), name_(name) {} + + bool ConfigChangeRequiresNewPolicyInstance( + LoadBalancingPolicy::Config* old_config, + LoadBalancingPolicy::Config* new_config) const override { + GPR_ASSERT(old_config->name() == name_); + GPR_ASSERT(new_config->name() == name_); + EdsLbConfig* old_eds_config = static_cast(old_config); + EdsLbConfig* new_eds_config = static_cast(new_config); + return old_eds_config->cluster_name() != new_eds_config->cluster_name() || + old_eds_config->eds_service_name() != + new_eds_config->eds_service_name(); + } + + OrphanablePtr CreateLoadBalancingPolicy( + const char* name, LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(name_, std::move(args)); + } + + private: + const char* name_; + }; + + const char* name_; +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_eds_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique(grpc_core::kEds)); + // TODO(roth): This is here just for backward compatibility with some + // old tests we have internally. Remove this once they are upgraded + // to use the new policy name and config. + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique(grpc_core::kXds)); +} + +void grpc_lb_policy_eds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc b/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc new file mode 100644 index 00000000000..b91d1b3bb9d --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc @@ -0,0 +1,524 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/ext/filters/client_channel/xds/xds_client.h" +#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" + +namespace grpc_core { + +TraceFlag grpc_lb_lrs_trace(false, "lrs_lb"); + +namespace { + +constexpr char kLrs[] = "lrs_experimental"; + +// Config for LRS LB policy. +class LrsLbConfig : public LoadBalancingPolicy::Config { + public: + LrsLbConfig(RefCountedPtr child_policy, + std::string cluster_name, std::string eds_service_name, + std::string lrs_load_reporting_server_name, + RefCountedPtr locality_name) + : child_policy_(std::move(child_policy)), + cluster_name_(std::move(cluster_name)), + eds_service_name_(std::move(eds_service_name)), + lrs_load_reporting_server_name_( + std::move(lrs_load_reporting_server_name)), + locality_name_(std::move(locality_name)) {} + + const char* name() const override { return kLrs; } + + RefCountedPtr child_policy() const { + return child_policy_; + } + const std::string& cluster_name() const { return cluster_name_; } + const std::string& eds_service_name() const { return eds_service_name_; } + const std::string& lrs_load_reporting_server_name() const { + return lrs_load_reporting_server_name_; + }; + RefCountedPtr locality_name() const { + return locality_name_; + } + + private: + RefCountedPtr child_policy_; + std::string cluster_name_; + std::string eds_service_name_; + std::string lrs_load_reporting_server_name_; + RefCountedPtr locality_name_; +}; + +// LRS LB policy. +class LrsLb : public LoadBalancingPolicy { + public: + LrsLb(RefCountedPtr xds_client, Args args); + + const char* name() const override { return kLrs; } + + void UpdateLocked(UpdateArgs args) override; + void ExitIdleLocked() override; + void ResetBackoffLocked() override; + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class RefCountedPicker : public RefCounted { + public: + explicit RefCountedPicker(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A picker that wraps the picker from the child to perform load reporting. + class LoadReportingPicker : public SubchannelPicker { + public: + LoadReportingPicker(RefCountedPtr picker, + RefCountedPtr locality_stats) + : picker_(std::move(picker)), + locality_stats_(std::move(locality_stats)) {} + + PickResult Pick(PickArgs args); + + private: + RefCountedPtr picker_; + RefCountedPtr locality_stats_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr lrs_policy) + : lrs_policy_(std::move(lrs_policy)) {} + + ~Helper() { lrs_policy_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr lrs_policy_; + }; + + ~LrsLb(); + + void ShutdownLocked() override; + + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + void UpdateChildPolicyLocked(ServerAddressList addresses, + const grpc_channel_args* args); + + void MaybeUpdatePickerLocked(); + + // Current config from the resolver. + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // The xds client. + RefCountedPtr xds_client_; + + // The stats for client-side load reporting. + RefCountedPtr locality_stats_; + + OrphanablePtr child_policy_; + + // Latest state and picker reported by the child policy. + grpc_connectivity_state state_ = GRPC_CHANNEL_IDLE; + RefCountedPtr picker_; +}; + +// +// LrsLb::LoadReportingPicker +// + +LoadBalancingPolicy::PickResult LrsLb::LoadReportingPicker::Pick( + LoadBalancingPolicy::PickArgs args) { + // Forward the pick to the picker returned from the child policy. + PickResult result = picker_->Pick(args); + if (result.type == PickResult::PICK_COMPLETE && + result.subchannel != nullptr) { + // Record a call started. + locality_stats_->AddCallStarted(); + // Intercept the recv_trailing_metadata op to record call completion. + XdsClusterLocalityStats* locality_stats = + locality_stats_->Ref(DEBUG_LOCATION, "LocalityStats+call").release(); + result.recv_trailing_metadata_ready = + // Note: This callback does not run in either the control plane + // combiner or in the data plane mutex. + [locality_stats](grpc_error* error, MetadataInterface* /*metadata*/, + CallState* /*call_state*/) { + const bool call_failed = error != GRPC_ERROR_NONE; + locality_stats->AddCallFinished(call_failed); + locality_stats->Unref(DEBUG_LOCATION, "LocalityStats+call"); + }; + } + return result; +} + +// +// LrsLb +// + +LrsLb::LrsLb(RefCountedPtr xds_client, Args args) + : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] created -- using xds client %p from channel", + this, xds_client_.get()); + } +} + +LrsLb::~LrsLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] destroying xds LB policy", this); + } +} + +void LrsLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] shutting down", this); + } + shutting_down_ = true; + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + if (child_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + interested_parties()); + child_policy_.reset(); + } + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_.reset(); + locality_stats_.reset(); + xds_client_.reset(); +} + +void LrsLb::ExitIdleLocked() { + if (child_policy_ != nullptr) child_policy_->ExitIdleLocked(); +} + +void LrsLb::ResetBackoffLocked() { + // The XdsClient will have its backoff reset by the xds resolver, so we + // don't need to do it here. + if (child_policy_ != nullptr) child_policy_->ResetBackoffLocked(); +} + +void LrsLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Received update", this); + } + // Update config. + auto old_config = std::move(config_); + config_ = std::move(args.config); + // Update load reporting if needed. + if (old_config == nullptr || + config_->lrs_load_reporting_server_name() != + old_config->lrs_load_reporting_server_name() || + config_->cluster_name() != old_config->cluster_name() || + config_->eds_service_name() != old_config->eds_service_name() || + *config_->locality_name() != *old_config->locality_name()) { + locality_stats_ = xds_client_->AddClusterLocalityStats( + config_->lrs_load_reporting_server_name(), config_->cluster_name(), + config_->eds_service_name(), config_->locality_name()); + MaybeUpdatePickerLocked(); + } + // Update child policy. + UpdateChildPolicyLocked(std::move(args.addresses), args.args); + args.args = nullptr; // Ownership passed to UpdateChildPolicyLocked(). +} + +void LrsLb::MaybeUpdatePickerLocked() { + if (picker_ != nullptr) { + auto lrs_picker = + absl::make_unique(picker_, locality_stats_); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] updating connectivity: state=%s picker=%p", + this, ConnectivityStateName(state_), lrs_picker.get()); + } + channel_control_helper()->UpdateState(state_, std::move(lrs_picker)); + } +} + +OrphanablePtr LrsLb::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_lrs_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Created new child policy handler %p", this, + lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void LrsLb::UpdateChildPolicyLocked(ServerAddressList addresses, + const grpc_channel_args* args) { + // Create policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(args); + } + // Construct update args. + UpdateArgs update_args; + update_args.addresses = std::move(addresses); + update_args.config = config_->child_policy(); + update_args.args = args; + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Updating child policy handler %p", this, + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +// +// LrsLb::Helper +// + +RefCountedPtr LrsLb::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (lrs_policy_->shutting_down_) return nullptr; + return lrs_policy_->channel_control_helper()->CreateSubchannel(args); +} + +void LrsLb::Helper::UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) { + if (lrs_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, + "[lrs_lb %p] child connectivity state update: state=%s picker=%p", + lrs_policy_.get(), ConnectivityStateName(state), picker.get()); + } + // Save the state and picker. + lrs_policy_->state_ = state; + lrs_policy_->picker_ = MakeRefCounted(std::move(picker)); + // Wrap the picker and return it to the channel. + lrs_policy_->MaybeUpdatePickerLocked(); +} + +void LrsLb::Helper::RequestReresolution() { + if (lrs_policy_->shutting_down_) return; + lrs_policy_->channel_control_helper()->RequestReresolution(); +} + +void LrsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { + if (lrs_policy_->shutting_down_) return; + lrs_policy_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// factory +// + +class LrsLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + RefCountedPtr xds_client = + XdsClient::GetFromChannelArgs(*args.args); + if (xds_client == nullptr) { + gpr_log(GPR_ERROR, + "XdsClient not present in channel args -- cannot instantiate " + "lrs LB policy"); + return nullptr; + } + return MakeOrphanable(std::move(xds_client), std::move(args)); + } + + const char* name() const override { return kLrs; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // lrs was mentioned as a policy in the deprecated loadBalancingPolicy + // field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:lrs policy requires configuration. " + "Please use loadBalancingConfig field of service config instead."); + return nullptr; + } + std::vector error_list; + // Child policy. + RefCountedPtr child_policy; + auto it = json.object_value().find("childPolicy"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:childPolicy error:required field missing")); + } else { + grpc_error* parse_error = GRPC_ERROR_NONE; + child_policy = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + it->second, &parse_error); + if (child_policy == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + std::vector child_errors; + child_errors.push_back(parse_error); + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); + } + } + // Cluster name. + std::string cluster_name; + it = json.object_value().find("clusterName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:type should be string")); + } else { + cluster_name = it->second.string_value(); + } + // EDS service name. + std::string eds_service_name; + it = json.object_value().find("edsServiceName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:edsServiceName error:type should be string")); + } else { + eds_service_name = it->second.string_value(); + } + } + // Locality. + RefCountedPtr locality_name; + it = json.object_value().find("locality"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:locality error:required field missing")); + } else { + std::vector child_errors = + ParseLocality(it->second, &locality_name); + if (!child_errors.empty()) { + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:locality", &child_errors)); + } + } + // LRS load reporting server name. + std::string lrs_load_reporting_server_name; + it = json.object_value().find("lrsLoadReportingServerName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:type should be string")); + } else { + lrs_load_reporting_server_name = it->second.string_value(); + } + if (!error_list.empty()) { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "lrs_experimental LB policy config", &error_list); + return nullptr; + } + return MakeRefCounted( + std::move(child_policy), std::move(cluster_name), + std::move(eds_service_name), std::move(lrs_load_reporting_server_name), + std::move(locality_name)); + } + + private: + static std::vector ParseLocality( + const Json& json, RefCountedPtr* name) { + std::vector error_list; + if (json.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "locality field is not an object")); + return error_list; + } + std::string region; + auto it = json.object_value().find("region"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"region\" field is not a string")); + } else { + region = it->second.string_value(); + } + } + std::string zone; + it = json.object_value().find("zone"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"zone\" field is not a string")); + } else { + zone = it->second.string_value(); + } + } + std::string subzone; + it = json.object_value().find("subzone"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"subzone\" field is not a string")); + } else { + subzone = it->second.string_value(); + } + } + if (region.empty() && zone.empty() && subzone.empty()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "at least one of region, zone, or subzone must be set")); + } + if (error_list.empty()) { + *name = MakeRefCounted(region, zone, subzone); + } + return error_list; + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_lrs_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_lrs_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc deleted file mode 100644 index 9196bf628ca..00000000000 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ /dev/null @@ -1,1754 +0,0 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/socket_utils.h" - -#include -#include -#include - -#include "absl/types/optional.h" - -#include -#include -#include -#include - -#include "src/core/ext/filters/client_channel/client_channel.h" -#include "src/core/ext/filters/client_channel/lb_policy.h" -#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h" -#include "src/core/ext/filters/client_channel/lb_policy_factory.h" -#include "src/core/ext/filters/client_channel/lb_policy_registry.h" -#include "src/core/ext/filters/client_channel/parse_address.h" -#include "src/core/ext/filters/client_channel/server_address.h" -#include "src/core/ext/filters/client_channel/service_config.h" -#include "src/core/ext/filters/client_channel/xds/xds_client.h" -#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" -#include "src/core/lib/backoff/backoff.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/map.h" -#include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/orphanable.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/combiner.h" -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/slice/slice_hash_table.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/channel_init.h" -#include "src/core/lib/transport/static_metadata.h" - -#define GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS 10000 -#define GRPC_XDS_DEFAULT_LOCALITY_RETENTION_INTERVAL_MS (15 * 60 * 1000) -#define GRPC_XDS_DEFAULT_FAILOVER_TIMEOUT_MS 10000 - -namespace grpc_core { - -TraceFlag grpc_lb_xds_trace(false, "xds_lb"); - -namespace { - -constexpr char kXds[] = "xds_experimental"; - -class XdsConfig : public LoadBalancingPolicy::Config { - public: - XdsConfig(RefCountedPtr child_policy, - RefCountedPtr fallback_policy, - std::string eds_service_name, - absl::optional lrs_load_reporting_server_name) - : child_policy_(std::move(child_policy)), - fallback_policy_(std::move(fallback_policy)), - eds_service_name_(std::move(eds_service_name)), - lrs_load_reporting_server_name_( - std::move(lrs_load_reporting_server_name)) {} - - const char* name() const override { return kXds; } - - RefCountedPtr child_policy() const { - return child_policy_; - } - - RefCountedPtr fallback_policy() const { - return fallback_policy_; - } - - const char* eds_service_name() const { - return eds_service_name_.empty() ? nullptr : eds_service_name_.c_str(); - }; - - const absl::optional& lrs_load_reporting_server_name() const { - return lrs_load_reporting_server_name_; - }; - - private: - RefCountedPtr child_policy_; - RefCountedPtr fallback_policy_; - std::string eds_service_name_; - absl::optional lrs_load_reporting_server_name_; -}; - -class XdsLb : public LoadBalancingPolicy { - public: - explicit XdsLb(Args args); - - const char* name() const override { return kXds; } - - void UpdateLocked(UpdateArgs args) override; - void ResetBackoffLocked() override; - - private: - class EndpointWatcher; - - // A simple wrapper to convert the picker returned from a locality's child - // policy as a unique_ptr<> to a RefCountedPtr<>. This allows it to be - // referenced by both the picker and the locality. - class RefCountedEndpointPicker : public RefCounted { - public: - explicit RefCountedEndpointPicker(std::unique_ptr picker) - : picker_(std::move(picker)) {} - PickResult Pick(PickArgs args) { return picker_->Pick(args); } - - private: - std::unique_ptr picker_; - }; - - // A picker that wraps the RefCountedEndpointPicker and performs load - // reporting for the locality. - class LoadReportingPicker : public RefCounted { - public: - LoadReportingPicker(RefCountedPtr picker, - RefCountedPtr locality_stats) - : picker_(std::move(picker)), - locality_stats_(std::move(locality_stats)) {} - - PickResult Pick(PickArgs args); - - RefCountedEndpointPicker* picker() const { return picker_.get(); } - XdsClusterLocalityStats* locality_stats() const { - return locality_stats_.get(); - } - - private: - RefCountedPtr picker_; - RefCountedPtr locality_stats_; - }; - - // A picker that uses a stateless weighting algorithm to pick the locality - // to use for each request. - class LocalityPicker : public SubchannelPicker { - public: - // Maintains a weighted list of pickers from each locality that is in ready - // state. The first element in the pair represents the end of a range - // proportional to the locality's weight. The start of the range is the - // previous value in the vector and is 0 for the first element. - using PickerList = - InlinedVector>, - 1>; - LocalityPicker(XdsLb* xds_policy, PickerList pickers) - : drop_stats_(xds_policy->drop_stats_), - drop_config_(xds_policy->drop_config_), - pickers_(std::move(pickers)) {} - - PickResult Pick(PickArgs args) override; - - private: - // Calls the picker of the locality that the key falls within. - PickResult PickFromLocality(const uint32_t key, PickArgs args); - - RefCountedPtr drop_stats_; - RefCountedPtr drop_config_; - PickerList pickers_; - }; - - class FallbackHelper : public ChannelControlHelper { - public: - explicit FallbackHelper(RefCountedPtr parent) - : parent_(std::move(parent)) {} - - ~FallbackHelper() { parent_.reset(DEBUG_LOCATION, "FallbackHelper"); } - - RefCountedPtr CreateSubchannel( - const grpc_channel_args& args) override; - void UpdateState(grpc_connectivity_state state, - std::unique_ptr picker) override; - void RequestReresolution() override; - void AddTraceEvent(TraceSeverity severity, StringView message) override; - - private: - RefCountedPtr parent_; - }; - - // Each LocalityMap holds a ref to the XdsLb. - class LocalityMap : public InternallyRefCounted { - public: - // Each Locality holds a ref to the LocalityMap it is in. - class Locality : public InternallyRefCounted { - public: - Locality(RefCountedPtr locality_map, - RefCountedPtr name); - ~Locality(); - - void UpdateLocked(uint32_t locality_weight, ServerAddressList serverlist, - bool update_locality_stats); - void ShutdownLocked(); - void ResetBackoffLocked(); - void DeactivateLocked(); - void Orphan() override; - - uint32_t weight() const { return weight_; } - - grpc_connectivity_state connectivity_state() const { - return connectivity_state_; - } - - RefCountedPtr GetLoadReportingPicker() { - // Recreate load reporting picker if stats object has changed. - if (load_reporting_picker_ == nullptr || - load_reporting_picker_->picker() != picker_wrapper_.get() || - load_reporting_picker_->locality_stats() != stats_.get()) { - load_reporting_picker_ = - MakeRefCounted(picker_wrapper_, stats_); - } - return load_reporting_picker_; - } - - void set_locality_map(RefCountedPtr locality_map) { - locality_map_ = std::move(locality_map); - } - - private: - class Helper : public ChannelControlHelper { - public: - explicit Helper(RefCountedPtr locality) - : locality_(std::move(locality)) {} - - ~Helper() { locality_.reset(DEBUG_LOCATION, "Helper"); } - - RefCountedPtr CreateSubchannel( - const grpc_channel_args& args) override; - void UpdateState(grpc_connectivity_state state, - std::unique_ptr picker) override; - // This is a no-op, because we get the addresses from the xds - // client, which is a watch-based API. - void RequestReresolution() override {} - void AddTraceEvent(TraceSeverity severity, StringView message) override; - - private: - RefCountedPtr locality_; - }; - - // Methods for dealing with the child policy. - OrphanablePtr CreateChildPolicyLocked( - const grpc_channel_args* args); - grpc_channel_args* CreateChildPolicyArgsLocked( - const grpc_channel_args* args); - - void UpdateLocalityStats(); - - static void OnDelayedRemovalTimer(void* arg, grpc_error* error); - static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); - - XdsLb* xds_policy() const { return locality_map_->xds_policy(); } - - // The owning locality map. - RefCountedPtr locality_map_; - - RefCountedPtr name_; - RefCountedPtr stats_; - OrphanablePtr child_policy_; - RefCountedPtr picker_wrapper_; - RefCountedPtr load_reporting_picker_; - grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_IDLE; - uint32_t weight_; - - // States for delayed removal. - grpc_timer delayed_removal_timer_; - grpc_closure on_delayed_removal_timer_; - bool delayed_removal_timer_callback_pending_ = false; - bool shutdown_ = false; - }; - - LocalityMap(RefCountedPtr xds_policy, uint32_t priority); - - ~LocalityMap() { xds_policy_.reset(DEBUG_LOCATION, "LocalityMap"); } - - void UpdateLocked( - const XdsApi::PriorityListUpdate::LocalityMap& priority_update, - bool update_locality_stats); - void ResetBackoffLocked(); - void UpdateXdsPickerLocked(); - OrphanablePtr ExtractLocalityLocked( - const RefCountedPtr& name); - void DeactivateLocked(); - // Returns true if this locality map becomes the currently used one (i.e., - // its priority is selected) after reactivation. - bool MaybeReactivateLocked(); - void MaybeCancelFailoverTimerLocked(); - - void Orphan() override; - - XdsLb* xds_policy() const { return xds_policy_.get(); } - uint32_t priority() const { return priority_; } - grpc_connectivity_state connectivity_state() const { - return connectivity_state_; - } - bool failover_timer_callback_pending() const { - return failover_timer_callback_pending_; - } - - private: - void OnLocalityStateUpdateLocked(); - void UpdateConnectivityStateLocked(); - static void OnDelayedRemovalTimer(void* arg, grpc_error* error); - static void OnFailoverTimer(void* arg, grpc_error* error); - static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); - static void OnFailoverTimerLocked(void* arg, grpc_error* error); - - const XdsApi::PriorityListUpdate& priority_list_update() const { - return xds_policy_->priority_list_update_; - } - const XdsApi::PriorityListUpdate::LocalityMap* locality_map_update() const { - return xds_policy_->priority_list_update_.Find(priority_); - } - - RefCountedPtr xds_policy_; - - std::map, OrphanablePtr, - XdsLocalityName::Less> - localities_; - const uint32_t priority_; - grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_IDLE; - - // States for delayed removal. - grpc_timer delayed_removal_timer_; - grpc_closure on_delayed_removal_timer_; - bool delayed_removal_timer_callback_pending_ = false; - - // States of failover. - grpc_timer failover_timer_; - grpc_closure on_failover_timer_; - bool failover_timer_callback_pending_ = false; - }; - - ~XdsLb(); - - void ShutdownLocked() override; - - const char* eds_service_name() const { - if (config_ != nullptr && config_->eds_service_name() != nullptr) { - return config_->eds_service_name(); - } - return server_name_.c_str(); - } - - XdsClient* xds_client() const { - return xds_client_from_channel_ != nullptr ? xds_client_from_channel_.get() - : xds_client_.get(); - } - - void UpdatePrioritiesLocked(bool update_locality_stats); - void UpdateXdsPickerLocked(); - void MaybeCreateLocalityMapLocked(uint32_t priority); - void FailoverOnConnectionFailureLocked(); - void FailoverOnDisconnectionLocked(uint32_t failed_priority); - void SwitchToHigherPriorityLocked(uint32_t priority); - void DeactivatePrioritiesLowerThan(uint32_t priority); - OrphanablePtr ExtractLocalityLocked( - const RefCountedPtr& name, uint32_t exclude_priority); - // Callers should make sure the priority list is non-empty. - uint32_t LowestPriority() const { - return static_cast(priorities_.size()) - 1; - } - bool Contains(uint32_t priority) { return priority < priorities_.size(); } - - // Methods for dealing with fallback state. - void MaybeCancelFallbackAtStartupChecks(); - static void OnFallbackTimer(void* arg, grpc_error* error); - static void OnFallbackTimerLocked(void* arg, grpc_error* error); - void UpdateFallbackPolicyLocked(); - OrphanablePtr CreateFallbackPolicyLocked( - const grpc_channel_args* args); - void MaybeExitFallbackMode(); - - // Server name from target URI. - std::string server_name_; - - // Current channel args and config from the resolver. - const grpc_channel_args* args_ = nullptr; - RefCountedPtr config_; - - // Internal state. - bool shutting_down_ = false; - - // The xds client and endpoint watcher. - // If we get the XdsClient from the channel, we store it in - // xds_client_from_channel_; if we create it ourselves, we store it in - // xds_client_. - RefCountedPtr xds_client_from_channel_; - OrphanablePtr xds_client_; - // A pointer to the endpoint watcher, to be used when cancelling the watch. - // Note that this is not owned, so this pointer must never be derefernced. - EndpointWatcher* endpoint_watcher_ = nullptr; - - // Whether the checks for fallback at startup are ALL pending. There are - // several cases where this can be reset: - // 1. The fallback timer fires, we enter fallback mode. - // 2. Before the fallback timer fires, the endpoint watcher reports an - // error, we enter fallback mode. - // 3. Before the fallback timer fires, if any child policy in the locality map - // becomes READY, we cancel the fallback timer. - bool fallback_at_startup_checks_pending_ = false; - // Timeout in milliseconds for before using fallback backend addresses. - // 0 means not using fallback. - const grpc_millis lb_fallback_timeout_ms_; - // The backend addresses from the resolver. - ServerAddressList fallback_backend_addresses_; - // Fallback timer. - grpc_timer lb_fallback_timer_; - grpc_closure lb_on_fallback_; - - // Non-null iff we are in fallback mode. - OrphanablePtr fallback_policy_; - - const grpc_millis locality_retention_interval_ms_; - const grpc_millis locality_map_failover_timeout_ms_; - // The list of locality maps, indexed by priority. P0 is the highest - // priority. - InlinedVector, 2> priorities_; - // The priority that is being used. - uint32_t current_priority_ = UINT32_MAX; - // The update for priority_list_. - XdsApi::PriorityListUpdate priority_list_update_; - - // The config for dropping calls. - RefCountedPtr drop_config_; - - // Drop stats for client-side load reporting. - RefCountedPtr drop_stats_; -}; - -// -// XdsLb::LoadReportingPicker -// - -LoadBalancingPolicy::PickResult XdsLb::LoadReportingPicker::Pick( - LoadBalancingPolicy::PickArgs args) { - // Forward the pick to the picker returned from the child policy. - PickResult result = picker_->Pick(args); - if (result.type != PickResult::PICK_COMPLETE || - result.subchannel == nullptr || locality_stats_ == nullptr) { - return result; - } - // Record a call started. - locality_stats_->AddCallStarted(); - // Intercept the recv_trailing_metadata op to record call completion. - XdsClusterLocalityStats* locality_stats = - locality_stats_->Ref(DEBUG_LOCATION, "LocalityStats+call").release(); - result.recv_trailing_metadata_ready = - // Note: This callback does not run in either the control plane - // combiner or in the data plane mutex. - [locality_stats](grpc_error* error, MetadataInterface* /*metadata*/, - CallState* /*call_state*/) { - const bool call_failed = error != GRPC_ERROR_NONE; - locality_stats->AddCallFinished(call_failed); - locality_stats->Unref(DEBUG_LOCATION, "LocalityStats+call"); - }; - return result; -} - -// -// XdsLb::LocalityPicker -// - -XdsLb::PickResult XdsLb::LocalityPicker::Pick(PickArgs args) { - // Handle drop. - const std::string* drop_category; - if (drop_config_->ShouldDrop(&drop_category)) { - if (drop_stats_ != nullptr) drop_stats_->AddCallDropped(*drop_category); - PickResult result; - result.type = PickResult::PICK_COMPLETE; - return result; - } - // If we didn't drop, we better have some localities to pick from. - if (pickers_.empty()) { // Should never happen. - PickResult result; - result.type = PickResult::PICK_FAILED; - result.error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "xds picker not given any localities"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); - return result; - } - // Generate a random number in [0, total weight). - const uint32_t key = rand() % pickers_[pickers_.size() - 1].first; - // Forward pick to whichever locality maps to the range in which the - // random number falls in. - return PickFromLocality(key, args); -} - -XdsLb::PickResult XdsLb::LocalityPicker::PickFromLocality(const uint32_t key, - PickArgs args) { - size_t mid = 0; - size_t start_index = 0; - size_t end_index = pickers_.size() - 1; - size_t index = 0; - while (end_index > start_index) { - mid = (start_index + end_index) / 2; - if (pickers_[mid].first > key) { - end_index = mid; - } else if (pickers_[mid].first < key) { - start_index = mid + 1; - } else { - index = mid + 1; - break; - } - } - if (index == 0) index = start_index; - GPR_ASSERT(pickers_[index].first > key); - return pickers_[index].second->Pick(args); -} - -// -// XdsLb::FallbackHelper -// - -RefCountedPtr XdsLb::FallbackHelper::CreateSubchannel( - const grpc_channel_args& args) { - if (parent_->shutting_down_) return nullptr; - return parent_->channel_control_helper()->CreateSubchannel(args); -} - -void XdsLb::FallbackHelper::UpdateState( - grpc_connectivity_state state, std::unique_ptr picker) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->UpdateState(state, std::move(picker)); -} - -void XdsLb::FallbackHelper::RequestReresolution() { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->RequestReresolution(); -} - -void XdsLb::FallbackHelper::AddTraceEvent(TraceSeverity severity, - StringView message) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->AddTraceEvent(severity, message); -} - -// -// XdsLb::EndpointWatcher -// - -class XdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { - public: - explicit EndpointWatcher(RefCountedPtr xds_policy) - : xds_policy_(std::move(xds_policy)) {} - - ~EndpointWatcher() { xds_policy_.reset(DEBUG_LOCATION, "EndpointWatcher"); } - - void OnEndpointChanged(XdsApi::EdsUpdate update) override { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Received EDS update from xds client", - xds_policy_.get()); - } - // If the balancer tells us to drop all the calls, we should exit fallback - // mode immediately. - if (update.drop_config->drop_all()) xds_policy_->MaybeExitFallbackMode(); - // Update the drop config. - const bool drop_config_changed = - xds_policy_->drop_config_ == nullptr || - *xds_policy_->drop_config_ != *update.drop_config; - xds_policy_->drop_config_ = std::move(update.drop_config); - // Ignore identical locality update. - if (xds_policy_->priority_list_update_ == update.priority_list_update) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Incoming locality update identical to current, " - "ignoring. (drop_config_changed=%d)", - xds_policy_.get(), drop_config_changed); - } - if (drop_config_changed) { - xds_policy_->UpdateXdsPickerLocked(); - } - return; - } - // Update the priority list. - xds_policy_->priority_list_update_ = std::move(update.priority_list_update); - xds_policy_->UpdatePrioritiesLocked(false /*update_locality_stats*/); - } - - void OnError(grpc_error* error) override { - // If the fallback-at-startup checks are pending, go into fallback mode - // immediately. This short-circuits the timeout for the - // fallback-at-startup case. - if (xds_policy_->fallback_at_startup_checks_pending_) { - gpr_log(GPR_INFO, - "[xdslb %p] xds watcher reported error; entering fallback " - "mode: %s", - xds_policy_.get(), grpc_error_string(error)); - xds_policy_->fallback_at_startup_checks_pending_ = false; - grpc_timer_cancel(&xds_policy_->lb_fallback_timer_); - xds_policy_->UpdateFallbackPolicyLocked(); - // If the xds call failed, request re-resolution. - // TODO(roth): We check the error string contents here to - // differentiate between the xds call failing and the xds channel - // going into TRANSIENT_FAILURE. This is a pretty ugly hack, - // but it's okay for now, since we're not yet sure whether we will - // continue to support the current fallback functionality. If we - // decide to keep the fallback approach, then we should either - // find a cleaner way to expose the difference between these two - // cases or decide that we're okay re-resolving in both cases. - // Note that even if we do keep the current fallback functionality, - // this re-resolution will only be necessary if we are going to be - // using this LB policy with resolvers other than the xds resolver. - if (strstr(grpc_error_string(error), "xds call failed")) { - xds_policy_->channel_control_helper()->RequestReresolution(); - } - } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] xds watcher reported error (ignoring): %s", - xds_policy_.get(), grpc_error_string(error)); - } - GRPC_ERROR_UNREF(error); - } - - private: - RefCountedPtr xds_policy_; -}; - -// -// ctor and dtor -// - -XdsLb::XdsLb(Args args) - : LoadBalancingPolicy(std::move(args)), - xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)), - lb_fallback_timeout_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, - {GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX})), - locality_retention_interval_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_LOCALITY_RETENTION_INTERVAL_MS, - {GRPC_XDS_DEFAULT_LOCALITY_RETENTION_INTERVAL_MS, 0, INT_MAX})), - locality_map_failover_timeout_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS, - {GRPC_XDS_DEFAULT_FAILOVER_TIMEOUT_MS, 0, INT_MAX})) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] created -- xds client from channel: %p", this, - xds_client_from_channel_.get()); - } - // Record server name. - const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI); - const char* server_uri = grpc_channel_arg_get_string(arg); - GPR_ASSERT(server_uri != nullptr); - grpc_uri* uri = grpc_uri_parse(server_uri, true); - GPR_ASSERT(uri->path[0] != '\0'); - server_name_ = uri->path[0] == '/' ? uri->path + 1 : uri->path; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] server name from channel: %s", this, - server_name_.c_str()); - } - grpc_uri_destroy(uri); -} - -XdsLb::~XdsLb() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] destroying xds LB policy", this); - } - grpc_channel_args_destroy(args_); -} - -void XdsLb::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] shutting down", this); - } - shutting_down_ = true; - MaybeCancelFallbackAtStartupChecks(); - priorities_.clear(); - drop_stats_.reset(); - if (fallback_policy_ != nullptr) { - grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(), - interested_parties()); - fallback_policy_.reset(); - } - // Cancel the endpoint watch here instead of in our dtor if we are using the - // XdsResolver, because the watcher holds a ref to us and we might not be - // destroying the Xds client leading to a situation where the Xds lb policy is - // never destroyed. - if (xds_client_from_channel_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] cancelling watch for %s", this, - eds_service_name()); - } - xds_client()->CancelEndpointDataWatch(StringView(eds_service_name()), - endpoint_watcher_); - xds_client_from_channel_.reset(); - } - xds_client_.reset(); -} - -// -// public methods -// - -void XdsLb::ResetBackoffLocked() { - // When the XdsClient is instantiated in the resolver instead of in this - // LB policy, this is done via the resolver, so we don't need to do it - // for xds_client_from_channel_ here. - if (xds_client_ != nullptr) xds_client_->ResetBackoff(); - for (size_t i = 0; i < priorities_.size(); ++i) { - priorities_[i]->ResetBackoffLocked(); - } - if (fallback_policy_ != nullptr) { - fallback_policy_->ResetBackoffLocked(); - } -} - -void XdsLb::UpdateLocked(UpdateArgs args) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Received update", this); - } - const bool is_initial_update = args_ == nullptr; - // Update config. - auto old_config = std::move(config_); - config_ = std::move(args.config); - // Update fallback address list. - fallback_backend_addresses_ = std::move(args.addresses); - // Update args. - grpc_channel_args_destroy(args_); - args_ = args.args; - args.args = nullptr; - // Update the existing fallback policy. The fallback policy config and/or the - // fallback addresses may be new. - if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked(); - if (is_initial_update) { - // Initialize XdsClient. - if (xds_client_from_channel_ == nullptr) { - grpc_error* error = GRPC_ERROR_NONE; - xds_client_ = MakeOrphanable( - combiner(), interested_parties(), StringView(eds_service_name()), - nullptr /* service config watcher */, *args_, &error); - // TODO(roth): If we decide that we care about fallback mode, add - // proper error handling here. - GPR_ASSERT(error == GRPC_ERROR_NONE); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Created xds client %p", this, - xds_client_.get()); - } - } - // Start fallback-at-startup checks. - grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_; - Ref(DEBUG_LOCATION, "on_fallback_timer").release(); // Held by closure - GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimer, this, - grpc_schedule_on_exec_ctx); - fallback_at_startup_checks_pending_ = true; - grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_); - } - // Update drop stats for load reporting if needed. - if (is_initial_update || config_->lrs_load_reporting_server_name() != - old_config->lrs_load_reporting_server_name()) { - drop_stats_.reset(); - if (config_->lrs_load_reporting_server_name().has_value()) { - drop_stats_ = xds_client()->AddClusterDropStats( - config_->lrs_load_reporting_server_name().value(), - // TODO(roth): We currently hard-code the assumption that - // cluster name and EDS service name are the same. Fix this - // as part of refectoring this LB policy. - eds_service_name(), eds_service_name()); - } - } - // On the initial update, create the endpoint watcher. - if (is_initial_update) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] starting watch for %s", this, - eds_service_name()); - } - auto watcher = absl::make_unique( - Ref(DEBUG_LOCATION, "EndpointWatcher")); - endpoint_watcher_ = watcher.get(); - xds_client()->WatchEndpointData(StringView(eds_service_name()), - std::move(watcher)); - } else { - // Update priority list. - // Note that this comes after updating drop_stats_, since we want that - // to be used by any new picker we create here. - // No need to do this on the initial update, since there won't be any - // priorities to update yet. - const bool update_locality_stats = - config_->lrs_load_reporting_server_name() != - old_config->lrs_load_reporting_server_name(); - UpdatePrioritiesLocked(update_locality_stats); - } -} - -// -// fallback-related methods -// - -void XdsLb::MaybeCancelFallbackAtStartupChecks() { - if (!fallback_at_startup_checks_pending_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Cancelling fallback timer", this); - } - grpc_timer_cancel(&lb_fallback_timer_); - fallback_at_startup_checks_pending_ = false; -} - -void XdsLb::OnFallbackTimer(void* arg, grpc_error* error) { - XdsLb* xdslb_policy = static_cast(arg); - xdslb_policy->combiner()->Run( - GRPC_CLOSURE_INIT(&xdslb_policy->lb_on_fallback_, - &XdsLb::OnFallbackTimerLocked, xdslb_policy, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) { - XdsLb* xdslb_policy = static_cast(arg); - // If some fallback-at-startup check is done after the timer fires but before - // this callback actually runs, don't fall back. - if (xdslb_policy->fallback_at_startup_checks_pending_ && - !xdslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) { - gpr_log(GPR_INFO, - "[xdslb %p] Child policy not ready after fallback timeout; " - "entering fallback mode", - xdslb_policy); - xdslb_policy->fallback_at_startup_checks_pending_ = false; - xdslb_policy->UpdateFallbackPolicyLocked(); - } - xdslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer"); -} - -void XdsLb::UpdateFallbackPolicyLocked() { - if (shutting_down_) return; - // Create policy if needed. - if (fallback_policy_ == nullptr) { - fallback_policy_ = CreateFallbackPolicyLocked(args_); - GPR_ASSERT(fallback_policy_ != nullptr); - } - // Perform update. - UpdateArgs update_args; - update_args.addresses = fallback_backend_addresses_; - update_args.config = config_->fallback_policy(); - update_args.args = grpc_channel_args_copy(args_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Updating fallback child policy handler %p", - this, fallback_policy_.get()); - } - fallback_policy_->UpdateLocked(std::move(update_args)); -} - -OrphanablePtr XdsLb::CreateFallbackPolicyLocked( - const grpc_channel_args* args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.combiner = combiner(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = - absl::make_unique(Ref(DEBUG_LOCATION, "FallbackHelper")); - OrphanablePtr lb_policy = - MakeOrphanable(std::move(lb_policy_args), - &grpc_lb_xds_trace); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Created new fallback child policy handler (%p)", this, - lb_policy.get()); - } - // Add the xDS's interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on xDS - // LB, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - interested_parties()); - return lb_policy; -} - -void XdsLb::MaybeExitFallbackMode() { - if (fallback_policy_ == nullptr) return; - gpr_log(GPR_INFO, "[xdslb %p] Exiting fallback mode", this); - fallback_policy_.reset(); -} - -// -// priority list-related methods -// - -void XdsLb::UpdatePrioritiesLocked(bool update_locality_stats) { - // 1. Remove from the priority list the priorities that are not in the update. - DeactivatePrioritiesLowerThan(priority_list_update_.LowestPriority()); - // 2. Update all the existing priorities. - for (uint32_t priority = 0; priority < priorities_.size(); ++priority) { - LocalityMap* locality_map = priorities_[priority].get(); - const auto* locality_map_update = priority_list_update_.Find(priority); - // If we have more current priorities than exist in the update, stop here. - if (locality_map_update == nullptr) break; - // Propagate locality_map_update. - // TODO(juanlishen): Find a clean way to skip duplicate update for a - // priority. - locality_map->UpdateLocked(*locality_map_update, update_locality_stats); - } - // 3. Only create a new locality map if all the existing ones have failed. - if (priorities_.empty() || - !priorities_[priorities_.size() - 1]->failover_timer_callback_pending()) { - const uint32_t new_priority = static_cast(priorities_.size()); - // Create a new locality map. Note that in some rare cases (e.g., the - // locality map reports TRANSIENT_FAILURE synchronously due to subchannel - // sharing), the following invocation may result in multiple locality maps - // to be created. - MaybeCreateLocalityMapLocked(new_priority); - } - // 4. If we updated locality stats and we already have at least one - // priority, update the picker to start using the new stats object(s). - if (update_locality_stats && !priorities_.empty()) { - UpdateXdsPickerLocked(); - } -} - -void XdsLb::UpdateXdsPickerLocked() { - // If we are in fallback mode, don't generate an xds picker from localities. - if (fallback_policy_ != nullptr) return; - // If we're dropping all calls, report READY, even though we won't - // have a selected priority. - if (drop_config_ != nullptr && drop_config_->drop_all()) { - channel_control_helper()->UpdateState( - GRPC_CHANNEL_READY, - absl::make_unique(this, LocalityPicker::PickerList{})); - return; - } - // If we don't have a selected priority, report TRANSIENT_FAILURE. - if (current_priority_ == UINT32_MAX) { - if (fallback_policy_ == nullptr) { - grpc_error* error = grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("no ready locality map"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, - absl::make_unique(error)); - } - return; - } - priorities_[current_priority_]->UpdateXdsPickerLocked(); -} - -void XdsLb::MaybeCreateLocalityMapLocked(uint32_t priority) { - // Exhausted priorities in the update. - if (!priority_list_update_.Contains(priority)) return; - auto new_locality_map = - new LocalityMap(Ref(DEBUG_LOCATION, "LocalityMap"), priority); - priorities_.emplace_back(OrphanablePtr(new_locality_map)); - new_locality_map->UpdateLocked(*priority_list_update_.Find(priority), - false /*update_locality_stats*/); -} - -void XdsLb::FailoverOnConnectionFailureLocked() { - const uint32_t failed_priority = LowestPriority(); - // If we're failing over from the lowest priority, report TRANSIENT_FAILURE. - if (failed_priority == priority_list_update_.LowestPriority()) { - UpdateXdsPickerLocked(); - } - MaybeCreateLocalityMapLocked(failed_priority + 1); -} - -void XdsLb::FailoverOnDisconnectionLocked(uint32_t failed_priority) { - current_priority_ = UINT32_MAX; - for (uint32_t next_priority = failed_priority + 1; - next_priority <= priority_list_update_.LowestPriority(); - ++next_priority) { - if (!Contains(next_priority)) { - MaybeCreateLocalityMapLocked(next_priority); - return; - } - if (priorities_[next_priority]->MaybeReactivateLocked()) return; - } -} - -void XdsLb::SwitchToHigherPriorityLocked(uint32_t priority) { - current_priority_ = priority; - DeactivatePrioritiesLowerThan(current_priority_); - UpdateXdsPickerLocked(); -} - -void XdsLb::DeactivatePrioritiesLowerThan(uint32_t priority) { - if (priorities_.empty()) return; - // Deactivate the locality maps from the lowest priority. - for (uint32_t p = LowestPriority(); p > priority; --p) { - if (locality_retention_interval_ms_ == 0) { - priorities_.pop_back(); - } else { - priorities_[p]->DeactivateLocked(); - } - } -} - -OrphanablePtr XdsLb::ExtractLocalityLocked( - const RefCountedPtr& name, uint32_t exclude_priority) { - for (uint32_t priority = 0; priority < priorities_.size(); ++priority) { - if (priority == exclude_priority) continue; - LocalityMap* locality_map = priorities_[priority].get(); - auto locality = locality_map->ExtractLocalityLocked(name); - if (locality != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] moving locality %p %s to new priority (%" PRIu32 - " -> %" PRIu32 ")", - this, locality.get(), name->AsHumanReadableString(), - exclude_priority, priority); - } - return locality; - } - } - return nullptr; -} - -// -// XdsLb::LocalityMap -// - -XdsLb::LocalityMap::LocalityMap(RefCountedPtr xds_policy, - uint32_t priority) - : xds_policy_(std::move(xds_policy)), priority_(priority) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Creating priority %" PRIu32, - xds_policy_.get(), priority_); - } - GRPC_CLOSURE_INIT(&on_failover_timer_, OnFailoverTimer, this, - grpc_schedule_on_exec_ctx); - // Start the failover timer. - Ref(DEBUG_LOCATION, "LocalityMap+OnFailoverTimerLocked").release(); - grpc_timer_init( - &failover_timer_, - ExecCtx::Get()->Now() + xds_policy_->locality_map_failover_timeout_ms_, - &on_failover_timer_); - failover_timer_callback_pending_ = true; - // This is the first locality map ever created, report CONNECTING. - if (priority_ == 0 && xds_policy_->fallback_policy_ == nullptr) { - xds_policy_->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, - absl::make_unique( - xds_policy_->Ref(DEBUG_LOCATION, "QueuePicker"))); - } -} - -void XdsLb::LocalityMap::UpdateLocked( - const XdsApi::PriorityListUpdate::LocalityMap& priority_update, - bool update_locality_stats) { - if (xds_policy_->shutting_down_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Start Updating priority %" PRIu32, - xds_policy(), priority_); - } - // Maybe reactivate the locality map in case all the active locality maps have - // failed. - MaybeReactivateLocked(); - // Remove (later) the localities not in priority_update. - for (auto iter = localities_.begin(); iter != localities_.end();) { - const auto& name = iter->first; - Locality* locality = iter->second.get(); - if (priority_update.Contains(name)) { - ++iter; - continue; - } - if (xds_policy()->locality_retention_interval_ms_ == 0) { - iter = localities_.erase(iter); - } else { - locality->DeactivateLocked(); - ++iter; - } - } - // Add or update the localities in priority_update. - for (const auto& p : priority_update.localities) { - const auto& name = p.first; - const auto& locality_update = p.second; - OrphanablePtr& locality = localities_[name]; - if (locality == nullptr) { - // Move from another locality map if possible. - locality = xds_policy_->ExtractLocalityLocked(name, priority_); - if (locality != nullptr) { - locality->set_locality_map( - Ref(DEBUG_LOCATION, "LocalityMap+Locality_move")); - } else { - locality = MakeOrphanable( - Ref(DEBUG_LOCATION, "LocalityMap+Locality"), name); - } - } - // Keep a copy of serverlist in the update so that we can compare it - // with the future ones. - locality->UpdateLocked(locality_update.lb_weight, - locality_update.serverlist, update_locality_stats); - } - // If this is the current priority and we removed all of the READY - // localities, go into state CONNECTING. - // TODO(roth): Ideally, we should model this as a graceful policy - // switch: we should keep using the old localities for a short period - // of time, long enough to give the new localities a chance to get - // connected. As part of refactoring this policy, we should try to - // fix that. - if (priority_ == xds_policy()->current_priority_) { - bool found_ready = false; - for (auto& p : localities_) { - const auto& locality_name = p.first; - Locality* locality = p.second.get(); - if (!locality_map_update()->Contains(locality_name)) continue; - if (locality->connectivity_state() == GRPC_CHANNEL_READY) { - found_ready = true; - break; - } - } - if (!found_ready) { - xds_policy_->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, - absl::make_unique( - xds_policy_->Ref(DEBUG_LOCATION, "QueuePicker"))); - xds_policy_->current_priority_ = UINT32_MAX; - } - } -} - -void XdsLb::LocalityMap::ResetBackoffLocked() { - for (auto& p : localities_) p.second->ResetBackoffLocked(); -} - -void XdsLb::LocalityMap::UpdateXdsPickerLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] constructing new picker", xds_policy()); - } - // Construct a new xds picker which maintains a map of all locality pickers - // that are ready. Each locality is represented by a portion of the range - // proportional to its weight, such that the total range is the sum of the - // weights of all localities. - LocalityPicker::PickerList picker_list; - uint32_t end = 0; - for (auto& p : localities_) { - const auto& locality_name = p.first; - Locality* locality = p.second.get(); - // Skip the localities that are not in the latest locality map update. - const auto* locality_update = locality_map_update(); - if (locality_update == nullptr) continue; - if (!locality_update->Contains(locality_name)) continue; - if (locality->connectivity_state() != GRPC_CHANNEL_READY) continue; - end += locality->weight(); - picker_list.push_back( - std::make_pair(end, locality->GetLoadReportingPicker())); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] locality=%s weight=%d picker=%p", - xds_policy(), locality_name->AsHumanReadableString(), - locality->weight(), picker_list.back().second.get()); - } - } - xds_policy()->channel_control_helper()->UpdateState( - GRPC_CHANNEL_READY, - absl::make_unique(xds_policy(), std::move(picker_list))); -} - -OrphanablePtr -XdsLb::LocalityMap::ExtractLocalityLocked( - const RefCountedPtr& name) { - for (auto iter = localities_.begin(); iter != localities_.end(); ++iter) { - const auto& name_in_map = iter->first; - if (*name_in_map == *name) { - auto locality = std::move(iter->second); - localities_.erase(iter); - return locality; - } - } - return nullptr; -} - -void XdsLb::LocalityMap::DeactivateLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] deactivating priority %" PRIu32, xds_policy(), - priority_); - } - // If already deactivated, don't do it again. - if (delayed_removal_timer_callback_pending_) return; - MaybeCancelFailoverTimerLocked(); - // Start a timer to delete the locality. - Ref(DEBUG_LOCATION, "LocalityMap+timer").release(); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Will remove priority %" PRIu32 " in %" PRId64 " ms.", - xds_policy(), priority_, - xds_policy()->locality_retention_interval_ms_); - } - GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, - grpc_schedule_on_exec_ctx); - grpc_timer_init( - &delayed_removal_timer_, - ExecCtx::Get()->Now() + xds_policy()->locality_retention_interval_ms_, - &on_delayed_removal_timer_); - delayed_removal_timer_callback_pending_ = true; -} - -bool XdsLb::LocalityMap::MaybeReactivateLocked() { - // Don't reactivate a priority that is not higher than the current one. - if (priority_ >= xds_policy_->current_priority_) return false; - // Reactivate this priority by cancelling deletion timer. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] reactivating priority %" PRIu32, xds_policy(), - priority_); - } - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - // Switch to this higher priority if it's READY. - if (connectivity_state_ != GRPC_CHANNEL_READY) return false; - xds_policy_->SwitchToHigherPriorityLocked(priority_); - return true; -} - -void XdsLb::LocalityMap::MaybeCancelFailoverTimerLocked() { - if (failover_timer_callback_pending_) grpc_timer_cancel(&failover_timer_); -} - -void XdsLb::LocalityMap::Orphan() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Priority %" PRIu32 " orphaned.", xds_policy(), - priority_); - } - MaybeCancelFailoverTimerLocked(); - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - localities_.clear(); - Unref(DEBUG_LOCATION, "LocalityMap+Orphan"); -} - -void XdsLb::LocalityMap::OnLocalityStateUpdateLocked() { - UpdateConnectivityStateLocked(); - // Ignore priorities not in priority_list_update. - if (!priority_list_update().Contains(priority_)) return; - const uint32_t current_priority = xds_policy_->current_priority_; - // Ignore lower-than-current priorities. - if (priority_ > current_priority) return; - // Maybe update fallback state. - if (connectivity_state_ == GRPC_CHANNEL_READY) { - xds_policy_->MaybeCancelFallbackAtStartupChecks(); - xds_policy_->MaybeExitFallbackMode(); - } - // Update is for a higher-than-current priority. (Special case: update is for - // any active priority if there is no current priority.) - if (priority_ < current_priority) { - if (connectivity_state_ == GRPC_CHANNEL_READY) { - MaybeCancelFailoverTimerLocked(); - // If a higher-than-current priority becomes READY, switch to use it. - xds_policy_->SwitchToHigherPriorityLocked(priority_); - } else if (connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // If a higher-than-current priority becomes TRANSIENT_FAILURE, only - // handle it if it's the priority that is still in failover timeout. - if (failover_timer_callback_pending_) { - MaybeCancelFailoverTimerLocked(); - xds_policy_->FailoverOnConnectionFailureLocked(); - } - } - return; - } - // Update is for current priority. - if (connectivity_state_ != GRPC_CHANNEL_READY) { - // Fail over if it's no longer READY. - xds_policy_->FailoverOnDisconnectionLocked(priority_); - } - // At this point, one of the following things has happened to the current - // priority. - // 1. It remained the same (but received picker update from its localities). - // 2. It changed to a lower priority due to failover. - // 3. It became invalid because failover didn't yield a READY priority. - // In any case, update the xds picker. - xds_policy_->UpdateXdsPickerLocked(); -} - -void XdsLb::LocalityMap::UpdateConnectivityStateLocked() { - size_t num_ready = 0; - size_t num_connecting = 0; - size_t num_idle = 0; - size_t num_transient_failures = 0; - for (const auto& p : localities_) { - const auto& locality_name = p.first; - const Locality* locality = p.second.get(); - // Skip the localities that are not in the latest locality map update. - if (!locality_map_update()->Contains(locality_name)) continue; - switch (locality->connectivity_state()) { - case GRPC_CHANNEL_READY: { - ++num_ready; - break; - } - case GRPC_CHANNEL_CONNECTING: { - ++num_connecting; - break; - } - case GRPC_CHANNEL_IDLE: { - ++num_idle; - break; - } - case GRPC_CHANNEL_TRANSIENT_FAILURE: { - ++num_transient_failures; - break; - } - default: - GPR_UNREACHABLE_CODE(return ); - } - } - if (num_ready > 0) { - connectivity_state_ = GRPC_CHANNEL_READY; - } else if (num_connecting > 0) { - connectivity_state_ = GRPC_CHANNEL_CONNECTING; - } else if (num_idle > 0) { - connectivity_state_ = GRPC_CHANNEL_IDLE; - } else { - connectivity_state_ = GRPC_CHANNEL_TRANSIENT_FAILURE; - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Priority %" PRIu32 " (%p) connectivity changed to %s", - xds_policy(), priority_, this, - ConnectivityStateName(connectivity_state_)); - } -} - -void XdsLb::LocalityMap::OnDelayedRemovalTimer(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->xds_policy_->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, - OnDelayedRemovalTimerLocked, self, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::OnDelayedRemovalTimerLocked(void* arg, - grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->delayed_removal_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->xds_policy_->shutting_down_) { - const bool keep = self->priority_list_update().Contains(self->priority_) && - self->priority_ <= self->xds_policy_->current_priority_; - if (!keep) { - // This check is to make sure we always delete the locality maps from - // the lowest priority even if the closures of the back-to-back timers - // are not run in FIFO order. - // TODO(juanlishen): Eliminate unnecessary maintenance overhead for some - // deactivated locality maps when out-of-order closures are run. - // TODO(juanlishen): Check the timer implementation to see if this - // defense is necessary. - if (self->priority_ == self->xds_policy_->LowestPriority()) { - self->xds_policy_->priorities_.pop_back(); - } else { - gpr_log(GPR_ERROR, - "[xdslb %p] Priority %" PRIu32 - " is not the lowest priority (highest numeric value) but is " - "attempted to be deleted.", - self->xds_policy(), self->priority_); - } - } - } - self->Unref(DEBUG_LOCATION, "LocalityMap+timer"); -} - -void XdsLb::LocalityMap::OnFailoverTimer(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->xds_policy_->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_failover_timer_, OnFailoverTimerLocked, self, - nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::OnFailoverTimerLocked(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->failover_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->xds_policy_->shutting_down_) { - self->xds_policy_->FailoverOnConnectionFailureLocked(); - } - self->Unref(DEBUG_LOCATION, "LocalityMap+OnFailoverTimerLocked"); -} - -// -// XdsLb::LocalityMap::Locality -// - -XdsLb::LocalityMap::Locality::Locality(RefCountedPtr locality_map, - RefCountedPtr name) - : locality_map_(std::move(locality_map)), name_(std::move(name)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] created Locality %p for %s", xds_policy(), - this, name_->AsHumanReadableString()); - } - // Initialize locality stats if load reporting is enabled. - UpdateLocalityStats(); -} - -XdsLb::LocalityMap::Locality::~Locality() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: destroying locality", - xds_policy(), this, name_->AsHumanReadableString()); - } - locality_map_.reset(DEBUG_LOCATION, "Locality"); -} - -void XdsLb::LocalityMap::Locality::UpdateLocalityStats() { - stats_.reset(); - if (xds_policy()->config_->lrs_load_reporting_server_name().has_value()) { - stats_ = xds_policy()->xds_client()->AddClusterLocalityStats( - xds_policy()->config_->lrs_load_reporting_server_name().value(), - // TODO(roth): We currently hard-code the assumption that - // cluster name and EDS service name are the same. Fix this - // as part of refectoring this LB policy. - xds_policy()->eds_service_name(), xds_policy()->eds_service_name(), - name_); - } -} - -grpc_channel_args* XdsLb::LocalityMap::Locality::CreateChildPolicyArgsLocked( - const grpc_channel_args* args_in) { - const grpc_arg args_to_add[] = { - // A channel arg indicating if the target is a backend inferred from a - // grpclb load balancer. - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER), - 1), - // Inhibit client-side health checking, since the balancer does - // this for us. - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1), - }; - return grpc_channel_args_copy_and_add(args_in, args_to_add, - GPR_ARRAY_SIZE(args_to_add)); -} - -OrphanablePtr -XdsLb::LocalityMap::Locality::CreateChildPolicyLocked( - const grpc_channel_args* args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.combiner = xds_policy()->combiner(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = - absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); - OrphanablePtr lb_policy = - MakeOrphanable(std::move(lb_policy_args), - &grpc_lb_xds_trace); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Locality %p %s: Created new child policy handler (%p)", - xds_policy(), this, name_->AsHumanReadableString(), - lb_policy.get()); - } - // Add the xDS's interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on - // xDS LB, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - xds_policy()->interested_parties()); - return lb_policy; -} - -void XdsLb::LocalityMap::Locality::UpdateLocked(uint32_t locality_weight, - ServerAddressList serverlist, - bool update_locality_stats) { - if (xds_policy()->shutting_down_) return; - // Update locality weight. - weight_ = locality_weight; - if (delayed_removal_timer_callback_pending_) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: reactivating", xds_policy(), - this, name_->AsHumanReadableString()); - } - grpc_timer_cancel(&delayed_removal_timer_); - } - // Update locality stats. - if (update_locality_stats) UpdateLocalityStats(); - // Construct update args. - UpdateArgs update_args; - update_args.addresses = std::move(serverlist); - update_args.config = xds_policy()->config_->child_policy(); - update_args.args = CreateChildPolicyArgsLocked(xds_policy()->args_); - // Create child policy if needed. - if (child_policy_ == nullptr) { - child_policy_ = CreateChildPolicyLocked(update_args.args); - GPR_ASSERT(child_policy_ != nullptr); - } - // Update the policy. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Locality %p %s: Updating child policy handler %p", - xds_policy(), this, name_->AsHumanReadableString(), - child_policy_.get()); - } - child_policy_->UpdateLocked(std::move(update_args)); -} - -void XdsLb::LocalityMap::Locality::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: shutting down locality", - xds_policy(), this, name_->AsHumanReadableString()); - } - stats_.reset(); - // Remove the child policy's interested_parties pollset_set from the - // xDS policy. - grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), - xds_policy()->interested_parties()); - child_policy_.reset(); - // Drop our ref to the child's picker, in case it's holding a ref to - // the child. - load_reporting_picker_.reset(); - picker_wrapper_.reset(); - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - shutdown_ = true; -} - -void XdsLb::LocalityMap::Locality::ResetBackoffLocked() { - child_policy_->ResetBackoffLocked(); -} - -void XdsLb::LocalityMap::Locality::Orphan() { - ShutdownLocked(); - Unref(); -} - -void XdsLb::LocalityMap::Locality::DeactivateLocked() { - // If already deactivated, don't do that again. - if (weight_ == 0) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: deactivating", xds_policy(), - this, name_->AsHumanReadableString()); - } - // Set the locality weight to 0 so that future xds picker won't contain this - // locality. - weight_ = 0; - // Start a timer to delete the locality. - Ref(DEBUG_LOCATION, "Locality+timer").release(); - GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, - grpc_schedule_on_exec_ctx); - grpc_timer_init( - &delayed_removal_timer_, - ExecCtx::Get()->Now() + xds_policy()->locality_retention_interval_ms_, - &on_delayed_removal_timer_); - delayed_removal_timer_callback_pending_ = true; -} - -void XdsLb::LocalityMap::Locality::OnDelayedRemovalTimer(void* arg, - grpc_error* error) { - Locality* self = static_cast(arg); - self->xds_policy()->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, - OnDelayedRemovalTimerLocked, self, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::Locality::OnDelayedRemovalTimerLocked( - void* arg, grpc_error* error) { - Locality* self = static_cast(arg); - self->delayed_removal_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->shutdown_ && self->weight_ == 0) { - self->locality_map_->localities_.erase(self->name_); - } - self->Unref(DEBUG_LOCATION, "Locality+timer"); -} - -// -// XdsLb::LocalityMap::Locality::Helper -// - -RefCountedPtr -XdsLb::LocalityMap::Locality::Helper::CreateSubchannel( - const grpc_channel_args& args) { - if (locality_->xds_policy()->shutting_down_) return nullptr; - return locality_->xds_policy()->channel_control_helper()->CreateSubchannel( - args); -} - -void XdsLb::LocalityMap::Locality::Helper::UpdateState( - grpc_connectivity_state state, std::unique_ptr picker) { - if (locality_->xds_policy()->shutting_down_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p helper %p] child policy handler %p reports state=%s", - locality_->xds_policy(), this, locality_->child_policy_.get(), - ConnectivityStateName(state)); - } - // Cache the state and picker in the locality. - locality_->connectivity_state_ = state; - locality_->picker_wrapper_ = - MakeRefCounted(std::move(picker)); - // Notify the locality map. - locality_->locality_map_->OnLocalityStateUpdateLocked(); -} - -void XdsLb::LocalityMap::Locality::Helper::AddTraceEvent(TraceSeverity severity, - StringView message) { - if (locality_->xds_policy()->shutting_down_) return; - locality_->xds_policy()->channel_control_helper()->AddTraceEvent(severity, - message); -} - -// -// factory -// - -class XdsFactory : public LoadBalancingPolicyFactory { - public: - OrphanablePtr CreateLoadBalancingPolicy( - LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args), &grpc_lb_xds_trace); - } - - const char* name() const override { return kXds; } - - RefCountedPtr ParseLoadBalancingConfig( - const Json& json, grpc_error** error) const override { - GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); - if (json.type() == Json::Type::JSON_NULL) { - // xds was mentioned as a policy in the deprecated loadBalancingPolicy - // field or in the client API. - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:loadBalancingPolicy error:xds policy requires configuration. " - "Please use loadBalancingConfig field of service config instead."); - return nullptr; - } - std::vector error_list; - // Child policy. - Json json_tmp; - const Json* child_policy_json; - auto it = json.object_value().find("childPolicy"); - if (it == json.object_value().end()) { - json_tmp = Json::Array{Json::Object{ - {"round_robin", Json::Object()}, - }}; - child_policy_json = &json_tmp; - } else { - child_policy_json = &it->second; - } - grpc_error* parse_error = GRPC_ERROR_NONE; - RefCountedPtr child_policy = - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - *child_policy_json, &parse_error); - if (child_policy == nullptr) { - GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); - std::vector child_errors; - child_errors.push_back(parse_error); - error_list.push_back( - GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); - } - // Fallback policy. - const Json* fallback_policy_json; - it = json.object_value().find("fallbackPolicy"); - if (it == json.object_value().end()) { - json_tmp = Json::Array{Json::Object{ - {"round_robin", Json::Object()}, - }}; - fallback_policy_json = &json_tmp; - } else { - fallback_policy_json = &it->second; - } - RefCountedPtr fallback_policy = - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - *fallback_policy_json, &parse_error); - if (fallback_policy == nullptr) { - GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); - std::vector child_errors; - child_errors.push_back(parse_error); - error_list.push_back( - GRPC_ERROR_CREATE_FROM_VECTOR("field:fallbackPolicy", &child_errors)); - } - // EDS service name. - const char* eds_service_name = nullptr; - it = json.object_value().find("edsServiceName"); - if (it != json.object_value().end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:edsServiceName error:type should be string")); - } else { - eds_service_name = it->second.string_value().c_str(); - } - } - // LRS load reporting server name. - const char* lrs_load_reporting_server_name = nullptr; - it = json.object_value().find("lrsLoadReportingServerName"); - if (it != json.object_value().end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:lrsLoadReportingServerName error:type should be string")); - } else { - lrs_load_reporting_server_name = it->second.string_value().c_str(); - } - } - if (error_list.empty()) { - absl::optional optional_lrs_load_reporting_server_name; - if (lrs_load_reporting_server_name != nullptr) { - optional_lrs_load_reporting_server_name.emplace( - std::string(lrs_load_reporting_server_name)); - } - return MakeRefCounted( - std::move(child_policy), std::move(fallback_policy), - eds_service_name == nullptr ? "" : eds_service_name, - std::move(optional_lrs_load_reporting_server_name)); - } else { - *error = GRPC_ERROR_CREATE_FROM_VECTOR("Xds Parser", &error_list); - return nullptr; - } - } - - private: - class XdsChildHandler : public ChildPolicyHandler { - public: - XdsChildHandler(Args args, TraceFlag* tracer) - : ChildPolicyHandler(std::move(args), tracer) {} - - bool ConfigChangeRequiresNewPolicyInstance( - LoadBalancingPolicy::Config* old_config, - LoadBalancingPolicy::Config* new_config) const override { - GPR_ASSERT(old_config->name() == kXds); - GPR_ASSERT(new_config->name() == kXds); - XdsConfig* old_xds_config = static_cast(old_config); - XdsConfig* new_xds_config = static_cast(new_config); - const char* old_eds_service_name = - old_xds_config->eds_service_name() == nullptr - ? "" - : old_xds_config->eds_service_name(); - const char* new_eds_service_name = - new_xds_config->eds_service_name() == nullptr - ? "" - : new_xds_config->eds_service_name(); - return strcmp(old_eds_service_name, new_eds_service_name) != 0; - } - - OrphanablePtr CreateLoadBalancingPolicy( - const char* name, LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args)); - } - }; -}; - -} // namespace - -} // namespace grpc_core - -// -// Plugin registration -// - -void grpc_lb_policy_xds_init() { - grpc_core::LoadBalancingPolicyRegistry::Builder:: - RegisterLoadBalancingPolicyFactory( - absl::make_unique()); -} - -void grpc_lb_policy_xds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.h b/src/core/ext/filters/client_channel/lb_policy/xds/xds.h index 13d3435da34..1de3472a3a0 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.h +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.h @@ -29,5 +29,4 @@ #define GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER \ "grpc.address_is_backend_from_xds_load_balancer" -#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_H \ - */ +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_H */ diff --git a/src/core/ext/filters/client_channel/lb_policy_registry.cc b/src/core/ext/filters/client_channel/lb_policy_registry.cc index 79e5f2f4926..498b158900e 100644 --- a/src/core/ext/filters/client_channel/lb_policy_registry.cc +++ b/src/core/ext/filters/client_channel/lb_policy_registry.cc @@ -35,6 +35,8 @@ class RegistryState { void RegisterLoadBalancingPolicyFactory( std::unique_ptr factory) { + gpr_log(GPR_DEBUG, "registering LB policy factory for \"%s\"", + factory->name()); for (size_t i = 0; i < factories_.size(); ++i) { GPR_ASSERT(strcmp(factories_[i]->name(), factory->name()) != 0); } diff --git a/src/core/ext/filters/client_channel/xds/xds_client_stats.h b/src/core/ext/filters/client_channel/xds/xds_client_stats.h index ab592ae6954..e0dd29d75ff 100644 --- a/src/core/ext/filters/client_channel/xds/xds_client_stats.h +++ b/src/core/ext/filters/client_channel/xds/xds_client_stats.h @@ -42,11 +42,7 @@ class XdsLocalityName : public RefCounted { struct Less { bool operator()(const XdsLocalityName* lhs, const XdsLocalityName* rhs) const { - int cmp_result = lhs->region_.compare(rhs->region_); - if (cmp_result != 0) return cmp_result < 0; - cmp_result = lhs->zone_.compare(rhs->zone_); - if (cmp_result != 0) return cmp_result < 0; - return lhs->sub_zone_.compare(rhs->sub_zone_) < 0; + return lhs->Compare(*rhs) < 0; } bool operator()(const RefCountedPtr& lhs, @@ -65,6 +61,18 @@ class XdsLocalityName : public RefCounted { sub_zone_ == other.sub_zone_; } + bool operator!=(const XdsLocalityName& other) const { + return !(*this == other); + } + + int Compare(const XdsLocalityName& other) const { + int cmp_result = region_.compare(other.region_); + if (cmp_result != 0) return cmp_result; + cmp_result = zone_.compare(other.zone_); + if (cmp_result != 0) return cmp_result; + return sub_zone_.compare(other.sub_zone_); + } + const std::string& region() const { return region_; } const std::string& zone() const { return zone_; } const std::string& sub_zone() const { return sub_zone_; } diff --git a/src/core/plugin_registry/grpc_plugin_registry.cc b/src/core/plugin_registry/grpc_plugin_registry.cc index 20ad526d837..b52c5be1719 100644 --- a/src/core/plugin_registry/grpc_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_plugin_registry.cc @@ -36,8 +36,14 @@ void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); void grpc_lb_policy_cds_init(void); void grpc_lb_policy_cds_shutdown(void); -void grpc_lb_policy_xds_init(void); -void grpc_lb_policy_xds_shutdown(void); +void grpc_lb_policy_eds_init(void); +void grpc_lb_policy_eds_shutdown(void); +void grpc_lb_policy_lrs_init(void); +void grpc_lb_policy_lrs_shutdown(void); +void grpc_lb_policy_priority_init(void); +void grpc_lb_policy_priority_shutdown(void); +void grpc_lb_policy_weighted_target_init(void); +void grpc_lb_policy_weighted_target_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -78,8 +84,14 @@ void grpc_register_built_in_plugins(void) { grpc_lb_policy_grpclb_shutdown); grpc_register_plugin(grpc_lb_policy_cds_init, grpc_lb_policy_cds_shutdown); - grpc_register_plugin(grpc_lb_policy_xds_init, - grpc_lb_policy_xds_shutdown); + grpc_register_plugin(grpc_lb_policy_eds_init, + grpc_lb_policy_eds_shutdown); + grpc_register_plugin(grpc_lb_policy_lrs_init, + grpc_lb_policy_lrs_shutdown); + grpc_register_plugin(grpc_lb_policy_priority_init, + grpc_lb_policy_priority_shutdown); + grpc_register_plugin(grpc_lb_policy_weighted_target_init, + grpc_lb_policy_weighted_target_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc index bfed2e22ddd..b1e442a07c0 100644 --- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc @@ -44,8 +44,14 @@ void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); void grpc_lb_policy_cds_init(void); void grpc_lb_policy_cds_shutdown(void); -void grpc_lb_policy_xds_init(void); -void grpc_lb_policy_xds_shutdown(void); +void grpc_lb_policy_eds_init(void); +void grpc_lb_policy_eds_shutdown(void); +void grpc_lb_policy_lrs_init(void); +void grpc_lb_policy_lrs_shutdown(void); +void grpc_lb_policy_priority_init(void); +void grpc_lb_policy_priority_shutdown(void); +void grpc_lb_policy_weighted_target_init(void); +void grpc_lb_policy_weighted_target_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -86,8 +92,14 @@ void grpc_register_built_in_plugins(void) { grpc_lb_policy_grpclb_shutdown); grpc_register_plugin(grpc_lb_policy_cds_init, grpc_lb_policy_cds_shutdown); - grpc_register_plugin(grpc_lb_policy_xds_init, - grpc_lb_policy_xds_shutdown); + grpc_register_plugin(grpc_lb_policy_eds_init, + grpc_lb_policy_eds_shutdown); + grpc_register_plugin(grpc_lb_policy_lrs_init, + grpc_lb_policy_lrs_shutdown); + grpc_register_plugin(grpc_lb_policy_priority_init, + grpc_lb_policy_priority_shutdown); + grpc_register_plugin(grpc_lb_policy_weighted_target_init, + grpc_lb_policy_weighted_target_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 46cce67f77e..6c2dfed4ae3 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -28,6 +28,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -36,9 +37,12 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', diff --git a/test/core/client_channel/service_config_test.cc b/test/core/client_channel/service_config_test.cc index d659fbeb51d..5cdb51341ab 100644 --- a/test/core/client_channel/service_config_test.cc +++ b/test/core/client_channel/service_config_test.cc @@ -464,7 +464,7 @@ TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigXds) { "{\n" " \"loadBalancingConfig\":[\n" " { \"does_not_exist\":{} },\n" - " { \"xds_experimental\":{ \"balancerName\": \"fake:///lb\" } }\n" + " { \"eds_experimental\":{ \"clusterName\": \"foo\" } }\n" " ]\n" "}"; grpc_error* error = GRPC_ERROR_NONE; @@ -474,7 +474,7 @@ TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigXds) { static_cast( svc_cfg->GetGlobalParsedConfig(0)); auto lb_config = parsed_config->parsed_lb_config(); - EXPECT_STREQ(lb_config->name(), "xds_experimental"); + EXPECT_STREQ(lb_config->name(), "eds_experimental"); } TEST_F(ClientChannelParserTest, UnknownLoadBalancingConfig) { @@ -544,14 +544,14 @@ TEST_F(ClientChannelParserTest, UnknownLoadBalancingPolicy) { } TEST_F(ClientChannelParserTest, LoadBalancingPolicyXdsNotAllowed) { - const char* test_json = "{\"loadBalancingPolicy\":\"xds_experimental\"}"; + const char* test_json = "{\"loadBalancingPolicy\":\"eds_experimental\"}"; grpc_error* error = GRPC_ERROR_NONE; auto svc_cfg = ServiceConfig::Create(test_json, &error); std::regex regex( "Service config parsing error.*referenced_errors.*" "Global Params.*referenced_errors.*" "Client channel global parser.*referenced_errors.*" - "field:loadBalancingPolicy error:xds_experimental requires " + "field:loadBalancingPolicy error:eds_experimental requires " "a config. Please use loadBalancingConfig instead."); VerifyRegexMatch(error, regex); } diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index e84692d2812..2efbc38a364 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1153,7 +1153,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam { args.SetInt(GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, fallback_timeout); } if (failover_timeout > 0) { - args.SetInt(GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS, failover_timeout); + args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout); } if (xds_resource_does_not_exist_timeout > 0) { args.SetInt(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS, @@ -1285,7 +1285,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { : kDefaultServiceConfigWithoutLoadReporting_; result.service_config = grpc_core::ServiceConfig::Create(service_config_json, &error); - GRPC_ERROR_UNREF(error); + ASSERT_NE(result.service_config.get(), nullptr); + ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error); grpc_arg arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg( lb_channel_response_generator == nullptr ? lb_channel_response_generator_.get() @@ -1317,7 +1318,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { grpc_error* error = GRPC_ERROR_NONE; result.service_config = grpc_core::ServiceConfig::Create(service_config_json, &error); - GRPC_ERROR_UNREF(error); + ASSERT_NE(result.service_config.get(), nullptr); + ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error); } if (lb_channel_response_generator == nullptr) { lb_channel_response_generator = lb_channel_response_generator_.get(); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index edbe236dc17..60e99d7b5a0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1091,6 +1091,8 @@ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/http_proxy.h \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy.h \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.h \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ @@ -1106,10 +1108,13 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ +src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 5c03ac61231..fef1fd11d4a 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -888,6 +888,8 @@ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/http_proxy.h \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy.h \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.h \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ @@ -903,10 +905,13 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ +src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ diff --git a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh index 6ed8856bf0d..a9a74eef377 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh @@ -48,7 +48,7 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py bazel build //src/python/grpcio_tests/tests_py3_only/interop:xds_interop_client -GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,xds_lb "$PYTHON" \ +GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ --test_case=all \ --project_id=grpc-testing \ diff --git a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh index d0a1338817b..943ba2b1981 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh @@ -48,7 +48,7 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py bazel build test/cpp/interop:xds_interop_client -GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,xds_lb "$PYTHON" \ +GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ --test_case=all \ --project_id=grpc-testing \