From bbe2587c39d60709358842f49aa46f91cc577ef7 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Tue, 15 Jan 2019 13:59:59 -0800 Subject: [PATCH] Enable per-channel subchannel pool --- BUILD | 8 +- CMakeLists.txt | 24 ++- Makefile | 24 ++- build.yaml | 8 +- config.m4 | 4 +- config.w32 | 4 +- gRPC-C++.podspec | 4 +- gRPC-Core.podspec | 12 +- grpc.gemspec | 8 +- grpc.gyp | 16 +- include/grpc/impl/codegen/grpc_types.h | 3 + package.xml | 8 +- .../client_channel/client_channel_plugin.cc | 6 +- .../client_channel/global_subchannel_pool.cc | 177 ++++++++++++++++++ .../client_channel/global_subchannel_pool.h | 68 +++++++ .../ext/filters/client_channel/lb_policy.cc | 2 + .../ext/filters/client_channel/lb_policy.h | 11 ++ .../client_channel/lb_policy/grpclb/grpclb.cc | 4 +- .../lb_policy/pick_first/pick_first.cc | 3 - .../lb_policy/round_robin/round_robin.cc | 3 - .../lb_policy/subchannel_list.h | 5 +- .../client_channel/lb_policy/xds/xds.cc | 4 +- .../client_channel/local_subchannel_pool.cc | 96 ++++++++++ .../client_channel/local_subchannel_pool.h | 56 ++++++ .../filters/client_channel/request_routing.cc | 16 +- .../filters/client_channel/request_routing.h | 6 +- .../ext/filters/client_channel/subchannel.cc | 44 +++-- .../ext/filters/client_channel/subchannel.h | 6 +- .../subchannel_pool_interface.cc | 97 ++++++++++ .../subchannel_pool_interface.h | 94 ++++++++++ src/python/grpcio/grpc_core_dependencies.py | 4 +- test/cpp/end2end/client_lb_end2end_test.cc | 63 +++++++ test/cpp/end2end/grpclb_end2end_test.cc | 3 +- tools/doxygen/Doxyfile.core.internal | 8 +- .../generated/sources_and_headers.json | 12 +- 35 files changed, 832 insertions(+), 79 deletions(-) create mode 100644 src/core/ext/filters/client_channel/global_subchannel_pool.cc create mode 100644 src/core/ext/filters/client_channel/global_subchannel_pool.h create mode 100644 src/core/ext/filters/client_channel/local_subchannel_pool.cc create mode 100644 src/core/ext/filters/client_channel/local_subchannel_pool.h create mode 100644 src/core/ext/filters/client_channel/subchannel_pool_interface.cc create mode 100644 src/core/ext/filters/client_channel/subchannel_pool_interface.h diff --git a/BUILD b/BUILD index 5f53bbc6f0b..03dc449cb02 100644 --- a/BUILD +++ b/BUILD @@ -1049,11 +1049,13 @@ grpc_cc_library( "src/core/ext/filters/client_channel/client_channel_factory.cc", "src/core/ext/filters/client_channel/client_channel_plugin.cc", "src/core/ext/filters/client_channel/connector.cc", + "src/core/ext/filters/client_channel/global_subchannel_pool.cc", "src/core/ext/filters/client_channel/health/health_check_client.cc", "src/core/ext/filters/client_channel/http_connect_handshaker.cc", "src/core/ext/filters/client_channel/http_proxy.cc", "src/core/ext/filters/client_channel/lb_policy.cc", "src/core/ext/filters/client_channel/lb_policy_registry.cc", + "src/core/ext/filters/client_channel/local_subchannel_pool.cc", "src/core/ext/filters/client_channel/parse_address.cc", "src/core/ext/filters/client_channel/proxy_mapper.cc", "src/core/ext/filters/client_channel/proxy_mapper_registry.cc", @@ -1064,7 +1066,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/retry_throttle.cc", "src/core/ext/filters/client_channel/server_address.cc", "src/core/ext/filters/client_channel/subchannel.cc", - "src/core/ext/filters/client_channel/subchannel_index.cc", + "src/core/ext/filters/client_channel/subchannel_pool_interface.cc", ], hdrs = [ "src/core/ext/filters/client_channel/backup_poller.h", @@ -1072,12 +1074,14 @@ grpc_cc_library( "src/core/ext/filters/client_channel/client_channel_channelz.h", "src/core/ext/filters/client_channel/client_channel_factory.h", "src/core/ext/filters/client_channel/connector.h", + "src/core/ext/filters/client_channel/global_subchannel_pool.h", "src/core/ext/filters/client_channel/health/health_check_client.h", "src/core/ext/filters/client_channel/http_connect_handshaker.h", "src/core/ext/filters/client_channel/http_proxy.h", "src/core/ext/filters/client_channel/lb_policy.h", "src/core/ext/filters/client_channel/lb_policy_factory.h", "src/core/ext/filters/client_channel/lb_policy_registry.h", + "src/core/ext/filters/client_channel/local_subchannel_pool.h", "src/core/ext/filters/client_channel/parse_address.h", "src/core/ext/filters/client_channel/proxy_mapper.h", "src/core/ext/filters/client_channel/proxy_mapper_registry.h", @@ -1089,7 +1093,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/retry_throttle.h", "src/core/ext/filters/client_channel/server_address.h", "src/core/ext/filters/client_channel/subchannel.h", - "src/core/ext/filters/client_channel/subchannel_index.h", + "src/core/ext/filters/client_channel/subchannel_pool_interface.h", ], language = "c++", deps = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 13d5aca6584..bb1caaaf565 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1212,11 +1212,13 @@ add_library(grpc src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -1227,7 +1229,7 @@ add_library(grpc src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/filters/client_channel/health/health.pb.c src/core/tsi/fake_transport_security.cc @@ -1566,11 +1568,13 @@ add_library(grpc_cronet src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -1581,7 +1585,7 @@ add_library(grpc_cronet src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/filters/client_channel/health/health.pb.c third_party/nanopb/pb_common.c @@ -1941,11 +1945,13 @@ add_library(grpc_test_util src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -1956,7 +1962,7 @@ add_library(grpc_test_util src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/filters/client_channel/health/health.pb.c third_party/nanopb/pb_common.c @@ -2264,11 +2270,13 @@ add_library(grpc_test_util_unsecure src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -2279,7 +2287,7 @@ add_library(grpc_test_util_unsecure src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/filters/client_channel/health/health.pb.c third_party/nanopb/pb_common.c @@ -2599,11 +2607,13 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -2614,7 +2624,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/filters/client_channel/health/health.pb.c third_party/nanopb/pb_common.c @@ -3455,11 +3465,13 @@ add_library(grpc++_cronet src/core/ext/filters/client_channel/client_channel_factory.cc src/core/ext/filters/client_channel/client_channel_plugin.cc src/core/ext/filters/client_channel/connector.cc + src/core/ext/filters/client_channel/global_subchannel_pool.cc src/core/ext/filters/client_channel/health/health_check_client.cc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc src/core/ext/filters/client_channel/lb_policy_registry.cc + src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc src/core/ext/filters/client_channel/proxy_mapper.cc src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -3470,7 +3482,7 @@ add_library(grpc++_cronet src/core/ext/filters/client_channel/retry_throttle.cc src/core/ext/filters/client_channel/server_address.cc src/core/ext/filters/client_channel/subchannel.cc - src/core/ext/filters/client_channel/subchannel_index.cc + src/core/ext/filters/client_channel/subchannel_pool_interface.cc src/core/ext/filters/deadline/deadline_filter.cc src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc diff --git a/Makefile b/Makefile index 00186d891c6..1a64c9e9683 100644 --- a/Makefile +++ b/Makefile @@ -3729,11 +3729,13 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -3744,7 +3746,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ src/core/tsi/fake_transport_security.cc \ @@ -4077,11 +4079,13 @@ LIBGRPC_CRONET_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -4092,7 +4096,7 @@ LIBGRPC_CRONET_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ third_party/nanopb/pb_common.c \ @@ -4445,11 +4449,13 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -4460,7 +4466,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ third_party/nanopb/pb_common.c \ @@ -4755,11 +4761,13 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -4770,7 +4778,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ third_party/nanopb/pb_common.c \ @@ -5064,11 +5072,13 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -5079,7 +5089,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ third_party/nanopb/pb_common.c \ @@ -5897,11 +5907,13 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -5912,7 +5924,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc \ src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc \ diff --git a/build.yaml b/build.yaml index c7b4d751731..8f310e0e59d 100644 --- a/build.yaml +++ b/build.yaml @@ -576,12 +576,14 @@ filegroups: - src/core/ext/filters/client_channel/client_channel_channelz.h - src/core/ext/filters/client_channel/client_channel_factory.h - src/core/ext/filters/client_channel/connector.h + - src/core/ext/filters/client_channel/global_subchannel_pool.h - src/core/ext/filters/client_channel/health/health_check_client.h - src/core/ext/filters/client_channel/http_connect_handshaker.h - src/core/ext/filters/client_channel/http_proxy.h - src/core/ext/filters/client_channel/lb_policy.h - src/core/ext/filters/client_channel/lb_policy_factory.h - src/core/ext/filters/client_channel/lb_policy_registry.h + - src/core/ext/filters/client_channel/local_subchannel_pool.h - src/core/ext/filters/client_channel/parse_address.h - src/core/ext/filters/client_channel/proxy_mapper.h - src/core/ext/filters/client_channel/proxy_mapper_registry.h @@ -593,7 +595,7 @@ filegroups: - src/core/ext/filters/client_channel/retry_throttle.h - src/core/ext/filters/client_channel/server_address.h - src/core/ext/filters/client_channel/subchannel.h - - src/core/ext/filters/client_channel/subchannel_index.h + - src/core/ext/filters/client_channel/subchannel_pool_interface.h src: - src/core/ext/filters/client_channel/backup_poller.cc - src/core/ext/filters/client_channel/channel_connectivity.cc @@ -602,11 +604,13 @@ filegroups: - src/core/ext/filters/client_channel/client_channel_factory.cc - src/core/ext/filters/client_channel/client_channel_plugin.cc - src/core/ext/filters/client_channel/connector.cc + - src/core/ext/filters/client_channel/global_subchannel_pool.cc - src/core/ext/filters/client_channel/health/health_check_client.cc - src/core/ext/filters/client_channel/http_connect_handshaker.cc - src/core/ext/filters/client_channel/http_proxy.cc - src/core/ext/filters/client_channel/lb_policy.cc - src/core/ext/filters/client_channel/lb_policy_registry.cc + - src/core/ext/filters/client_channel/local_subchannel_pool.cc - src/core/ext/filters/client_channel/parse_address.cc - src/core/ext/filters/client_channel/proxy_mapper.cc - src/core/ext/filters/client_channel/proxy_mapper_registry.cc @@ -617,7 +621,7 @@ filegroups: - src/core/ext/filters/client_channel/retry_throttle.cc - src/core/ext/filters/client_channel/server_address.cc - src/core/ext/filters/client_channel/subchannel.cc - - src/core/ext/filters/client_channel/subchannel_index.cc + - src/core/ext/filters/client_channel/subchannel_pool_interface.cc plugin: grpc_client_channel uses: - grpc_base diff --git a/config.m4 b/config.m4 index ccb218a1200..46597e6f0e3 100644 --- a/config.m4 +++ b/config.m4 @@ -345,11 +345,13 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/client_channel_factory.cc \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ + src/core/ext/filters/client_channel/global_subchannel_pool.cc \ src/core/ext/filters/client_channel/health/health_check_client.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ + src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/proxy_mapper.cc \ src/core/ext/filters/client_channel/proxy_mapper_registry.cc \ @@ -360,7 +362,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/retry_throttle.cc \ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/subchannel.cc \ - src/core/ext/filters/client_channel/subchannel_index.cc \ + src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/client_channel/health/health.pb.c \ src/core/tsi/fake_transport_security.cc \ diff --git a/config.w32 b/config.w32 index fd48ec6f485..00b92e88a05 100644 --- a/config.w32 +++ b/config.w32 @@ -320,11 +320,13 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\client_channel_factory.cc " + "src\\core\\ext\\filters\\client_channel\\client_channel_plugin.cc " + "src\\core\\ext\\filters\\client_channel\\connector.cc " + + "src\\core\\ext\\filters\\client_channel\\global_subchannel_pool.cc " + "src\\core\\ext\\filters\\client_channel\\health\\health_check_client.cc " + "src\\core\\ext\\filters\\client_channel\\http_connect_handshaker.cc " + "src\\core\\ext\\filters\\client_channel\\http_proxy.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy_registry.cc " + + "src\\core\\ext\\filters\\client_channel\\local_subchannel_pool.cc " + "src\\core\\ext\\filters\\client_channel\\parse_address.cc " + "src\\core\\ext\\filters\\client_channel\\proxy_mapper.cc " + "src\\core\\ext\\filters\\client_channel\\proxy_mapper_registry.cc " + @@ -335,7 +337,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\retry_throttle.cc " + "src\\core\\ext\\filters\\client_channel\\server_address.cc " + "src\\core\\ext\\filters\\client_channel\\subchannel.cc " + - "src\\core\\ext\\filters\\client_channel\\subchannel_index.cc " + + "src\\core\\ext\\filters\\client_channel\\subchannel_pool_interface.cc " + "src\\core\\ext\\filters\\deadline\\deadline_filter.cc " + "src\\core\\ext\\filters\\client_channel\\health\\health.pb.c " + "src\\core\\tsi\\fake_transport_security.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index bf124304487..481892b63c7 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -346,12 +346,14 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', 'src/core/ext/filters/client_channel/connector.h', + 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', 'src/core/ext/filters/client_channel/lb_policy_factory.h', 'src/core/ext/filters/client_channel/lb_policy_registry.h', + 'src/core/ext/filters/client_channel/local_subchannel_pool.h', 'src/core/ext/filters/client_channel/parse_address.h', 'src/core/ext/filters/client_channel/proxy_mapper.h', 'src/core/ext/filters/client_channel/proxy_mapper_registry.h', @@ -363,7 +365,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/retry_throttle.h', 'src/core/ext/filters/client_channel/server_address.h', 'src/core/ext/filters/client_channel/subchannel.h', - 'src/core/ext/filters/client_channel/subchannel_index.h', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.h', 'src/core/ext/filters/deadline/deadline_filter.h', 'src/core/ext/filters/client_channel/health/health.pb.h', 'src/core/tsi/fake_transport_security.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 60f34ebd6a2..5bb6a514bb9 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -340,12 +340,14 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', 'src/core/ext/filters/client_channel/connector.h', + 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', 'src/core/ext/filters/client_channel/lb_policy_factory.h', 'src/core/ext/filters/client_channel/lb_policy_registry.h', + 'src/core/ext/filters/client_channel/local_subchannel_pool.h', 'src/core/ext/filters/client_channel/parse_address.h', 'src/core/ext/filters/client_channel/proxy_mapper.h', 'src/core/ext/filters/client_channel/proxy_mapper_registry.h', @@ -357,7 +359,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/retry_throttle.h', 'src/core/ext/filters/client_channel/server_address.h', 'src/core/ext/filters/client_channel/subchannel.h', - 'src/core/ext/filters/client_channel/subchannel_index.h', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.h', 'src/core/ext/filters/deadline/deadline_filter.h', 'src/core/ext/filters/client_channel/health/health.pb.h', 'src/core/tsi/fake_transport_security.h', @@ -785,11 +787,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -800,7 +804,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'src/core/tsi/fake_transport_security.cc', @@ -964,12 +968,14 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/client_channel_channelz.h', 'src/core/ext/filters/client_channel/client_channel_factory.h', 'src/core/ext/filters/client_channel/connector.h', + 'src/core/ext/filters/client_channel/global_subchannel_pool.h', 'src/core/ext/filters/client_channel/health/health_check_client.h', 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', 'src/core/ext/filters/client_channel/lb_policy_factory.h', 'src/core/ext/filters/client_channel/lb_policy_registry.h', + 'src/core/ext/filters/client_channel/local_subchannel_pool.h', 'src/core/ext/filters/client_channel/parse_address.h', 'src/core/ext/filters/client_channel/proxy_mapper.h', 'src/core/ext/filters/client_channel/proxy_mapper_registry.h', @@ -981,7 +987,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/retry_throttle.h', 'src/core/ext/filters/client_channel/server_address.h', 'src/core/ext/filters/client_channel/subchannel.h', - 'src/core/ext/filters/client_channel/subchannel_index.h', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.h', 'src/core/ext/filters/deadline/deadline_filter.h', 'src/core/ext/filters/client_channel/health/health.pb.h', 'src/core/tsi/fake_transport_security.h', diff --git a/grpc.gemspec b/grpc.gemspec index 60c5bc480b6..5e5eb65ed2f 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -276,12 +276,14 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/client_channel_channelz.h ) s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.h ) s.files += %w( src/core/ext/filters/client_channel/connector.h ) + s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.h ) s.files += %w( src/core/ext/filters/client_channel/health/health_check_client.h ) s.files += %w( src/core/ext/filters/client_channel/http_connect_handshaker.h ) s.files += %w( src/core/ext/filters/client_channel/http_proxy.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_factory.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_registry.h ) + s.files += %w( src/core/ext/filters/client_channel/local_subchannel_pool.h ) s.files += %w( src/core/ext/filters/client_channel/parse_address.h ) s.files += %w( src/core/ext/filters/client_channel/proxy_mapper.h ) s.files += %w( src/core/ext/filters/client_channel/proxy_mapper_registry.h ) @@ -293,7 +295,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/retry_throttle.h ) s.files += %w( src/core/ext/filters/client_channel/server_address.h ) s.files += %w( src/core/ext/filters/client_channel/subchannel.h ) - s.files += %w( src/core/ext/filters/client_channel/subchannel_index.h ) + s.files += %w( src/core/ext/filters/client_channel/subchannel_pool_interface.h ) s.files += %w( src/core/ext/filters/deadline/deadline_filter.h ) s.files += %w( src/core/ext/filters/client_channel/health/health.pb.h ) s.files += %w( src/core/tsi/fake_transport_security.h ) @@ -724,11 +726,13 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/client_channel_factory.cc ) s.files += %w( src/core/ext/filters/client_channel/client_channel_plugin.cc ) s.files += %w( src/core/ext/filters/client_channel/connector.cc ) + s.files += %w( src/core/ext/filters/client_channel/global_subchannel_pool.cc ) s.files += %w( src/core/ext/filters/client_channel/health/health_check_client.cc ) s.files += %w( src/core/ext/filters/client_channel/http_connect_handshaker.cc ) s.files += %w( src/core/ext/filters/client_channel/http_proxy.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_registry.cc ) + s.files += %w( src/core/ext/filters/client_channel/local_subchannel_pool.cc ) s.files += %w( src/core/ext/filters/client_channel/parse_address.cc ) s.files += %w( src/core/ext/filters/client_channel/proxy_mapper.cc ) s.files += %w( src/core/ext/filters/client_channel/proxy_mapper_registry.cc ) @@ -739,7 +743,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/retry_throttle.cc ) s.files += %w( src/core/ext/filters/client_channel/server_address.cc ) s.files += %w( src/core/ext/filters/client_channel/subchannel.cc ) - s.files += %w( src/core/ext/filters/client_channel/subchannel_index.cc ) + s.files += %w( src/core/ext/filters/client_channel/subchannel_pool_interface.cc ) s.files += %w( src/core/ext/filters/deadline/deadline_filter.cc ) s.files += %w( src/core/ext/filters/client_channel/health/health.pb.c ) s.files += %w( src/core/tsi/fake_transport_security.cc ) diff --git a/grpc.gyp b/grpc.gyp index 1f5b6384975..060af57efff 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -527,11 +527,13 @@ 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -542,7 +544,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'src/core/tsi/fake_transport_security.cc', @@ -789,11 +791,13 @@ 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -804,7 +808,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'third_party/nanopb/pb_common.c', @@ -1032,11 +1036,13 @@ 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -1047,7 +1053,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'third_party/nanopb/pb_common.c', @@ -1287,11 +1293,13 @@ 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -1302,7 +1310,7 @@ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 5d577eb8557..f9929186d58 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -355,6 +355,9 @@ typedef struct { * is 10000. Setting this to "0" will disable c-ares query timeouts * entirely. */ #define GRPC_ARG_DNS_ARES_QUERY_TIMEOUT_MS "grpc.dns_ares_query_timeout" +/** If set, uses a local subchannel pool within the channel. Otherwise, uses the + * global subchannel pool. */ +#define GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL "grpc.use_local_subchannel_pool" /** gRPC Objective-C channel pooling domain string. */ #define GRPC_ARG_CHANNEL_POOL_DOMAIN "grpc.channel_pooling_domain" /** gRPC Objective-C channel pooling id. */ diff --git a/package.xml b/package.xml index 81a4aabdf5a..523f78f1db6 100644 --- a/package.xml +++ b/package.xml @@ -281,12 +281,14 @@ + + @@ -298,7 +300,7 @@ - + @@ -729,11 +731,13 @@ + + @@ -744,7 +748,7 @@ - + diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.cc b/src/core/ext/filters/client_channel/client_channel_plugin.cc index e0784b7e5c1..2031ab449f5 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.cc +++ b/src/core/ext/filters/client_channel/client_channel_plugin.cc @@ -26,13 +26,13 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/ext/filters/client_channel/client_channel_channelz.h" +#include "src/core/ext/filters/client_channel/global_subchannel_pool.h" #include "src/core/ext/filters/client_channel/http_connect_handshaker.h" #include "src/core/ext/filters/client_channel/http_proxy.h" #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h" #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/ext/filters/client_channel/retry_throttle.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/surface/channel_init.h" static bool append_filter(grpc_channel_stack_builder* builder, void* arg) { @@ -54,7 +54,7 @@ void grpc_client_channel_init(void) { grpc_core::internal::ServerRetryThrottleMap::Init(); grpc_proxy_mapper_registry_init(); grpc_register_http_proxy_mapper(); - grpc_subchannel_index_init(); + grpc_core::GlobalSubchannelPool::Init(); grpc_channel_init_register_stage( GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, append_filter, (void*)&grpc_client_channel_filter); @@ -62,7 +62,7 @@ void grpc_client_channel_init(void) { } void grpc_client_channel_shutdown(void) { - grpc_subchannel_index_shutdown(); + grpc_core::GlobalSubchannelPool::Shutdown(); grpc_channel_init_shutdown(); grpc_proxy_mapper_registry_shutdown(); grpc_core::internal::ServerRetryThrottleMap::Shutdown(); diff --git a/src/core/ext/filters/client_channel/global_subchannel_pool.cc b/src/core/ext/filters/client_channel/global_subchannel_pool.cc new file mode 100644 index 00000000000..a41d993fe66 --- /dev/null +++ b/src/core/ext/filters/client_channel/global_subchannel_pool.cc @@ -0,0 +1,177 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include + +#include "src/core/ext/filters/client_channel/global_subchannel_pool.h" + +#include "src/core/ext/filters/client_channel/subchannel.h" + +namespace grpc_core { + +GlobalSubchannelPool::GlobalSubchannelPool() { + subchannel_map_ = grpc_avl_create(&subchannel_avl_vtable_); + gpr_mu_init(&mu_); +} + +GlobalSubchannelPool::~GlobalSubchannelPool() { + gpr_mu_destroy(&mu_); + grpc_avl_unref(subchannel_map_, nullptr); +} + +void GlobalSubchannelPool::Init() { + instance_ = New>( + MakeRefCounted()); +} + +void GlobalSubchannelPool::Shutdown() { + // To ensure Init() was called before. + GPR_ASSERT(instance_ != nullptr); + // To ensure Shutdown() was not called before. + GPR_ASSERT(*instance_ != nullptr); + instance_->reset(); + Delete(instance_); +} + +RefCountedPtr GlobalSubchannelPool::instance() { + GPR_ASSERT(instance_ != nullptr); + GPR_ASSERT(*instance_ != nullptr); + return *instance_; +} + +grpc_subchannel* GlobalSubchannelPool::RegisterSubchannel( + SubchannelKey* key, grpc_subchannel* constructed) { + grpc_subchannel* c = nullptr; + // Compare and swap (CAS) loop: + while (c == nullptr) { + // Ref the shared map to have a local copy. + gpr_mu_lock(&mu_); + grpc_avl old_map = grpc_avl_ref(subchannel_map_, nullptr); + gpr_mu_unlock(&mu_); + // Check to see if a subchannel already exists. + c = static_cast(grpc_avl_get(old_map, key, nullptr)); + if (c != nullptr) { + // The subchannel already exists. Reuse it. + c = GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(c, "subchannel_register+reuse"); + GRPC_SUBCHANNEL_UNREF(constructed, "subchannel_register+found_existing"); + // Exit the CAS loop without modifying the shared map. + } else { + // There hasn't been such subchannel. Add one. + // Note that we should ref the old map first because grpc_avl_add() will + // unref it while we still need to access it later. + grpc_avl new_map = grpc_avl_add( + grpc_avl_ref(old_map, nullptr), New(*key), + GRPC_SUBCHANNEL_WEAK_REF(constructed, "subchannel_register+new"), + nullptr); + // Try to publish the change to the shared map. It may happen (but + // unlikely) that some other thread has changed the shared map, so compare + // to make sure it's unchanged before swapping. Retry if it's changed. + gpr_mu_lock(&mu_); + if (old_map.root == subchannel_map_.root) { + GPR_SWAP(grpc_avl, new_map, subchannel_map_); + c = constructed; + } + gpr_mu_unlock(&mu_); + grpc_avl_unref(new_map, nullptr); + } + grpc_avl_unref(old_map, nullptr); + } + return c; +} + +void GlobalSubchannelPool::UnregisterSubchannel(SubchannelKey* key) { + bool done = false; + // Compare and swap (CAS) loop: + while (!done) { + // Ref the shared map to have a local copy. + gpr_mu_lock(&mu_); + grpc_avl old_map = grpc_avl_ref(subchannel_map_, nullptr); + gpr_mu_unlock(&mu_); + // Remove the subchannel. + // Note that we should ref the old map first because grpc_avl_remove() will + // unref it while we still need to access it later. + grpc_avl new_map = + grpc_avl_remove(grpc_avl_ref(old_map, nullptr), key, nullptr); + // Try to publish the change to the shared map. It may happen (but + // unlikely) that some other thread has changed the shared map, so compare + // to make sure it's unchanged before swapping. Retry if it's changed. + gpr_mu_lock(&mu_); + if (old_map.root == subchannel_map_.root) { + GPR_SWAP(grpc_avl, new_map, subchannel_map_); + done = true; + } + gpr_mu_unlock(&mu_); + grpc_avl_unref(new_map, nullptr); + grpc_avl_unref(old_map, nullptr); + } +} + +grpc_subchannel* GlobalSubchannelPool::FindSubchannel(SubchannelKey* key) { + // Lock, and take a reference to the subchannel map. + // We don't need to do the search under a lock as AVL's are immutable. + gpr_mu_lock(&mu_); + grpc_avl index = grpc_avl_ref(subchannel_map_, nullptr); + gpr_mu_unlock(&mu_); + grpc_subchannel* c = GRPC_SUBCHANNEL_REF_FROM_WEAK_REF( + static_cast(grpc_avl_get(index, key, nullptr)), + "found_from_pool"); + grpc_avl_unref(index, nullptr); + return c; +} + +RefCountedPtr* GlobalSubchannelPool::instance_ = nullptr; + +namespace { + +void sck_avl_destroy(void* p, void* user_data) { + SubchannelKey* key = static_cast(p); + Delete(key); +} + +void* sck_avl_copy(void* p, void* unused) { + const SubchannelKey* key = static_cast(p); + auto* new_key = New(*key); + return static_cast(new_key); +} + +long sck_avl_compare(void* a, void* b, void* unused) { + const SubchannelKey* key_a = static_cast(a); + const SubchannelKey* key_b = static_cast(b); + return key_a->Cmp(*key_b); +} + +void scv_avl_destroy(void* p, void* user_data) { + GRPC_SUBCHANNEL_WEAK_UNREF((grpc_subchannel*)p, "global_subchannel_pool"); +} + +void* scv_avl_copy(void* p, void* unused) { + GRPC_SUBCHANNEL_WEAK_REF((grpc_subchannel*)p, "global_subchannel_pool"); + return p; +} + +} // namespace + +const grpc_avl_vtable GlobalSubchannelPool::subchannel_avl_vtable_ = { + sck_avl_destroy, // destroy_key + sck_avl_copy, // copy_key + sck_avl_compare, // compare_keys + scv_avl_destroy, // destroy_value + scv_avl_copy // copy_value +}; + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/global_subchannel_pool.h b/src/core/ext/filters/client_channel/global_subchannel_pool.h new file mode 100644 index 00000000000..0deb3769360 --- /dev/null +++ b/src/core/ext/filters/client_channel/global_subchannel_pool.h @@ -0,0 +1,68 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_GLOBAL_SUBCHANNEL_POOL_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_GLOBAL_SUBCHANNEL_POOL_H + +#include + +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" + +namespace grpc_core { + +// The global subchannel pool. It shares subchannels among channels. There +// should be only one instance of this class. Init() should be called once at +// the filter initialization time; Shutdown() should be called once at the +// filter shutdown time. +// TODO(juanlishen): Enable subchannel retention. +class GlobalSubchannelPool final : public SubchannelPoolInterface { + public: + // The ctor and dtor are not intended to use directly. + GlobalSubchannelPool(); + ~GlobalSubchannelPool() override; + + // Should be called exactly once at filter initialization time. + static void Init(); + // Should be called exactly once at filter shutdown time. + static void Shutdown(); + + // Gets the singleton instance. + static RefCountedPtr instance(); + + // Implements interface methods. + grpc_subchannel* RegisterSubchannel(SubchannelKey* key, + grpc_subchannel* constructed) override; + void UnregisterSubchannel(SubchannelKey* key) override; + grpc_subchannel* FindSubchannel(SubchannelKey* key) override; + + private: + // The singleton instance. (It's a pointer to RefCountedPtr so that this + // non-local static object can be trivially destructible.) + static RefCountedPtr* instance_; + + // The vtable for subchannel operations in an AVL tree. + static const grpc_avl_vtable subchannel_avl_vtable_; + // A map from subchannel key to subchannel. + grpc_avl subchannel_map_; + // To protect subchannel_map_. + gpr_mu mu_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_GLOBAL_SUBCHANNEL_POOL_H */ diff --git a/src/core/ext/filters/client_channel/lb_policy.cc b/src/core/ext/filters/client_channel/lb_policy.cc index b4e803689e9..31b0399d874 100644 --- a/src/core/ext/filters/client_channel/lb_policy.cc +++ b/src/core/ext/filters/client_channel/lb_policy.cc @@ -19,6 +19,7 @@ #include #include "src/core/ext/filters/client_channel/lb_policy.h" + #include "src/core/lib/iomgr/combiner.h" grpc_core::DebugOnlyTraceFlag grpc_trace_lb_policy_refcount( @@ -30,6 +31,7 @@ LoadBalancingPolicy::LoadBalancingPolicy(const Args& args) : InternallyRefCounted(&grpc_trace_lb_policy_refcount), combiner_(GRPC_COMBINER_REF(args.combiner, "lb_policy")), client_channel_factory_(args.client_channel_factory), + subchannel_pool_(*args.subchannel_pool), interested_parties_(grpc_pollset_set_create()), request_reresolution_(nullptr) {} diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 293d8e960cf..b9d97e092af 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -24,6 +24,7 @@ #include "src/core/ext/filters/client_channel/client_channel_channelz.h" #include "src/core/ext/filters/client_channel/client_channel_factory.h" #include "src/core/ext/filters/client_channel/subchannel.h" +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" #include "src/core/lib/gprpp/abstract.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -53,6 +54,8 @@ class LoadBalancingPolicy : public InternallyRefCounted { grpc_combiner* combiner = nullptr; /// Used to create channels and subchannels. grpc_client_channel_factory* client_channel_factory = nullptr; + /// Subchannel pool. + RefCountedPtr* subchannel_pool; /// Channel args from the resolver. /// Note that the LB policy gets the set of addresses from the /// GRPC_ARG_SERVER_ADDRESS_LIST channel arg. @@ -171,6 +174,12 @@ class LoadBalancingPolicy : public InternallyRefCounted { grpc_pollset_set* interested_parties() const { return interested_parties_; } + /// Returns a pointer to the subchannel pool of type + /// RefCountedPtr. + RefCountedPtr* subchannel_pool() { + return &subchannel_pool_; + } + GRPC_ABSTRACT_BASE_CLASS protected: @@ -204,6 +213,8 @@ class LoadBalancingPolicy : public InternallyRefCounted { grpc_combiner* combiner_; /// Client channel factory, used to create channels and subchannels. grpc_client_channel_factory* client_channel_factory_; + /// Subchannel pool. + RefCountedPtr subchannel_pool_; /// Owned pointer to interested parties in load balancing decisions. grpc_pollset_set* interested_parties_; /// Callback to force a re-resolution. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index ba40febd534..40bf9c65644 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -85,7 +85,6 @@ #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" #include "src/core/ext/filters/client_channel/server_address.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" @@ -988,7 +987,6 @@ GrpcLb::GrpcLb(const LoadBalancingPolicy::Args& args) 1000)) { // Initialization. gpr_mu_init(&lb_channel_mu_); - grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&lb_channel_on_connectivity_changed_, &GrpcLb::OnBalancerChannelConnectivityChangedLocked, this, grpc_combiner_scheduler(args.combiner)); @@ -1032,7 +1030,6 @@ GrpcLb::~GrpcLb() { if (serverlist_ != nullptr) { grpc_grpclb_destroy_serverlist(serverlist_); } - grpc_subchannel_index_unref(); } void GrpcLb::ShutdownLocked() { @@ -1699,6 +1696,7 @@ void GrpcLb::CreateOrUpdateRoundRobinPolicyLocked() { lb_policy_args.combiner = combiner(); lb_policy_args.client_channel_factory = client_channel_factory(); lb_policy_args.args = args; + lb_policy_args.subchannel_pool = subchannel_pool(); CreateRoundRobinPolicyLocked(lb_policy_args); } grpc_channel_args_destroy(args); diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index d6ff74ec7f7..75eacb2e17e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -26,7 +26,6 @@ #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/server_address.h" #include "src/core/ext/filters/client_channel/subchannel.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/iomgr/combiner.h" @@ -164,7 +163,6 @@ PickFirst::PickFirst(const Args& args) : LoadBalancingPolicy(args) { gpr_log(GPR_INFO, "Pick First %p created.", this); } UpdateLocked(*args.args, args.lb_config); - grpc_subchannel_index_ref(); } PickFirst::~PickFirst() { @@ -176,7 +174,6 @@ PickFirst::~PickFirst() { GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); GPR_ASSERT(pending_picks_ == nullptr); grpc_connectivity_state_destroy(&state_tracker_); - grpc_subchannel_index_unref(); } void PickFirst::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 3bcb33ef11c..5143c6d8380 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -33,7 +33,6 @@ #include "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h" #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/subchannel.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/mutex_lock.h" @@ -221,7 +220,6 @@ RoundRobin::RoundRobin(const Args& args) : LoadBalancingPolicy(args) { gpr_log(GPR_INFO, "[RR %p] Created with %" PRIuPTR " subchannels", this, subchannel_list_->num_subchannels()); } - grpc_subchannel_index_ref(); } RoundRobin::~RoundRobin() { @@ -233,7 +231,6 @@ RoundRobin::~RoundRobin() { GPR_ASSERT(latest_pending_subchannel_list_ == nullptr); GPR_ASSERT(pending_picks_ == nullptr); grpc_connectivity_state_destroy(&state_tracker_); - grpc_subchannel_index_unref(); } void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 1d0ecbe3f64..55f5d6da85a 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -514,6 +514,9 @@ SubchannelList::SubchannelList( // policy, which does not use a SubchannelList. GPR_ASSERT(!addresses[i].IsBalancer()); InlinedVector args_to_add; + args_to_add.emplace_back(SubchannelPoolInterface::CreateChannelArg( + policy_->subchannel_pool()->get())); + const size_t subchannel_address_arg_index = args_to_add.size(); args_to_add.emplace_back( grpc_create_subchannel_address_arg(&addresses[i].address())); if (addresses[i].args() != nullptr) { @@ -524,7 +527,7 @@ SubchannelList::SubchannelList( grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove( &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), args_to_add.data(), args_to_add.size()); - gpr_free(args_to_add[0].value.string); + gpr_free(args_to_add[subchannel_address_arg_index].value.string); grpc_subchannel* subchannel = grpc_client_channel_factory_create_subchannel( client_channel_factory, new_args); grpc_channel_args_destroy(new_args); diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc index 8787f5bcc24..63bd8be011b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc @@ -80,7 +80,6 @@ #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" #include "src/core/ext/filters/client_channel/server_address.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" @@ -905,7 +904,6 @@ XdsLb::XdsLb(const LoadBalancingPolicy::Args& args) .set_max_backoff(GRPC_XDS_RECONNECT_MAX_BACKOFF_SECONDS * 1000)) { // Initialization. gpr_mu_init(&lb_channel_mu_); - grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&lb_channel_on_connectivity_changed_, &XdsLb::OnBalancerChannelConnectivityChangedLocked, this, grpc_combiner_scheduler(args.combiner)); @@ -949,7 +947,6 @@ XdsLb::~XdsLb() { if (serverlist_ != nullptr) { xds_grpclb_destroy_serverlist(serverlist_); } - grpc_subchannel_index_unref(); } void XdsLb::ShutdownLocked() { @@ -1526,6 +1523,7 @@ void XdsLb::CreateOrUpdateChildPolicyLocked() { LoadBalancingPolicy::Args lb_policy_args; lb_policy_args.combiner = combiner(); lb_policy_args.client_channel_factory = client_channel_factory(); + lb_policy_args.subchannel_pool = subchannel_pool(); lb_policy_args.args = args; CreateChildPolicyLocked(lb_policy_args); if (grpc_lb_xds_trace.enabled()) { diff --git a/src/core/ext/filters/client_channel/local_subchannel_pool.cc b/src/core/ext/filters/client_channel/local_subchannel_pool.cc new file mode 100644 index 00000000000..145fa4e0374 --- /dev/null +++ b/src/core/ext/filters/client_channel/local_subchannel_pool.cc @@ -0,0 +1,96 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include + +#include "src/core/ext/filters/client_channel/local_subchannel_pool.h" + +#include "src/core/ext/filters/client_channel/subchannel.h" + +namespace grpc_core { + +LocalSubchannelPool::LocalSubchannelPool() { + subchannel_map_ = grpc_avl_create(&subchannel_avl_vtable_); +} + +LocalSubchannelPool::~LocalSubchannelPool() { + grpc_avl_unref(subchannel_map_, nullptr); +} + +grpc_subchannel* LocalSubchannelPool::RegisterSubchannel( + SubchannelKey* key, grpc_subchannel* constructed) { + // Check to see if a subchannel already exists. + grpc_subchannel* c = static_cast( + grpc_avl_get(subchannel_map_, key, nullptr)); + if (c != nullptr) { + // The subchannel already exists. Reuse it. + c = GRPC_SUBCHANNEL_REF(c, "subchannel_register+reuse"); + GRPC_SUBCHANNEL_UNREF(constructed, "subchannel_register+found_existing"); + } else { + // There hasn't been such subchannel. Add one. + subchannel_map_ = grpc_avl_add(subchannel_map_, New(*key), + constructed, nullptr); + c = constructed; + } + return c; +} + +void LocalSubchannelPool::UnregisterSubchannel(SubchannelKey* key) { + subchannel_map_ = grpc_avl_remove(subchannel_map_, key, nullptr); +} + +grpc_subchannel* LocalSubchannelPool::FindSubchannel(SubchannelKey* key) { + grpc_subchannel* c = static_cast( + grpc_avl_get(subchannel_map_, key, nullptr)); + return c == nullptr ? c : GRPC_SUBCHANNEL_REF(c, "found_from_pool"); +} + +namespace { + +void sck_avl_destroy(void* p, void* user_data) { + SubchannelKey* key = static_cast(p); + Delete(key); +} + +void* sck_avl_copy(void* p, void* unused) { + const SubchannelKey* key = static_cast(p); + auto new_key = New(*key); + return static_cast(new_key); +} + +long sck_avl_compare(void* a, void* b, void* unused) { + const SubchannelKey* key_a = static_cast(a); + const SubchannelKey* key_b = static_cast(b); + return key_a->Cmp(*key_b); +} + +void scv_avl_destroy(void* p, void* user_data) {} + +void* scv_avl_copy(void* p, void* unused) { return p; } + +} // namespace + +const grpc_avl_vtable LocalSubchannelPool::subchannel_avl_vtable_ = { + sck_avl_destroy, // destroy_key + sck_avl_copy, // copy_key + sck_avl_compare, // compare_keys + scv_avl_destroy, // destroy_value + scv_avl_copy // copy_value +}; + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/local_subchannel_pool.h b/src/core/ext/filters/client_channel/local_subchannel_pool.h new file mode 100644 index 00000000000..9929cdb3627 --- /dev/null +++ b/src/core/ext/filters/client_channel/local_subchannel_pool.h @@ -0,0 +1,56 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LOCAL_SUBCHANNEL_POOL_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LOCAL_SUBCHANNEL_POOL_H + +#include + +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" + +namespace grpc_core { + +// The local subchannel pool that is owned by a single channel. It doesn't +// support subchannel sharing with other channels by nature. Nor does it support +// subchannel retention when a subchannel is not used. The only real purpose of +// using this subchannel pool is to allow subchannel reuse within the channel +// when an incoming resolver update contains some addresses for which the +// channel has already created subchannels. +// Thread-unsafe. +class LocalSubchannelPool final : public SubchannelPoolInterface { + public: + LocalSubchannelPool(); + ~LocalSubchannelPool() override; + + // Implements interface methods. + // Thread-unsafe. Intended to be invoked within the client_channel combiner. + grpc_subchannel* RegisterSubchannel(SubchannelKey* key, + grpc_subchannel* constructed) override; + void UnregisterSubchannel(SubchannelKey* key) override; + grpc_subchannel* FindSubchannel(SubchannelKey* key) override; + + private: + // The vtable for subchannel operations in an AVL tree. + static const grpc_avl_vtable subchannel_avl_vtable_; + // A map from subchannel key to subchannel. + grpc_avl subchannel_map_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LOCAL_SUBCHANNEL_POOL_H */ diff --git a/src/core/ext/filters/client_channel/request_routing.cc b/src/core/ext/filters/client_channel/request_routing.cc index f9a7e164e75..5e52456859e 100644 --- a/src/core/ext/filters/client_channel/request_routing.cc +++ b/src/core/ext/filters/client_channel/request_routing.cc @@ -32,8 +32,10 @@ #include #include "src/core/ext/filters/client_channel/backup_poller.h" +#include "src/core/ext/filters/client_channel/global_subchannel_pool.h" #include "src/core/ext/filters/client_channel/http_connect_handshaker.h" #include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/ext/filters/client_channel/local_subchannel_pool.h" #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h" #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/ext/filters/client_channel/retry_throttle.h" @@ -517,6 +519,14 @@ RequestRouter::RequestRouter( tracer_(tracer), process_resolver_result_(process_resolver_result), process_resolver_result_user_data_(process_resolver_result_user_data) { + // Get subchannel pool. + const grpc_arg* arg = + grpc_channel_args_find(args, GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL); + if (grpc_channel_arg_get_bool(arg, false)) { + subchannel_pool_ = MakeRefCounted(); + } else { + subchannel_pool_ = GlobalSubchannelPool::instance(); + } GRPC_CLOSURE_INIT(&on_resolver_result_changed_, &RequestRouter::OnResolverResultChangedLocked, this, grpc_combiner_scheduler(combiner)); @@ -666,6 +676,7 @@ void RequestRouter::CreateNewLbPolicyLocked( LoadBalancingPolicy::Args lb_policy_args; lb_policy_args.combiner = combiner_; lb_policy_args.client_channel_factory = client_channel_factory_; + lb_policy_args.subchannel_pool = &subchannel_pool_; lb_policy_args.args = resolver_result_; lb_policy_args.lb_config = lb_config; OrphanablePtr new_lb_policy = @@ -751,9 +762,8 @@ void RequestRouter::ConcatenateAndAddChannelTraceLocked( char* flat; size_t flat_len = 0; flat = gpr_strvec_flatten(&v, &flat_len); - channelz_node_->AddTraceEvent( - grpc_core::channelz::ChannelTrace::Severity::Info, - grpc_slice_new(flat, flat_len, gpr_free)); + channelz_node_->AddTraceEvent(channelz::ChannelTrace::Severity::Info, + grpc_slice_new(flat, flat_len, gpr_free)); gpr_strvec_destroy(&v); } } diff --git a/src/core/ext/filters/client_channel/request_routing.h b/src/core/ext/filters/client_channel/request_routing.h index 0c671229c8e..0027163869e 100644 --- a/src/core/ext/filters/client_channel/request_routing.h +++ b/src/core/ext/filters/client_channel/request_routing.h @@ -25,6 +25,7 @@ #include "src/core/ext/filters/client_channel/client_channel_factory.h" #include "src/core/ext/filters/client_channel/lb_policy.h" #include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/debug/trace.h" @@ -126,7 +127,7 @@ class RequestRouter { LoadBalancingPolicy* lb_policy() const { return lb_policy_.get(); } private: - using TraceStringVector = grpc_core::InlinedVector; + using TraceStringVector = InlinedVector; class ReresolutionRequestHandler; class LbConnectivityWatcher; @@ -169,6 +170,9 @@ class RequestRouter { OrphanablePtr lb_policy_; bool exit_idle_when_lb_policy_arrives_ = false; + // Subchannel pool to pass to LB policy. + RefCountedPtr subchannel_pool_; + grpc_connectivity_state_tracker state_tracker_; }; diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 640a052e91e..0c75ee046d9 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -33,7 +33,7 @@ #include "src/core/ext/filters/client_channel/health/health_check_client.h" #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h" -#include "src/core/ext/filters/client_channel/subchannel_index.h" +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" @@ -80,6 +80,9 @@ class ConnectedSubchannelStateWatcher; } // namespace grpc_core struct grpc_subchannel { + /** The subchannel pool this subchannel is in */ + grpc_core::RefCountedPtr subchannel_pool; + grpc_connector* connector; /** refcount @@ -92,7 +95,7 @@ struct grpc_subchannel { /** channel arguments */ grpc_channel_args* args; - grpc_subchannel_key* key; + grpc_core::SubchannelKey* key; /** set during connection */ grpc_connect_out_args connecting_result; @@ -375,7 +378,7 @@ static void subchannel_destroy(void* arg, grpc_error* error) { grpc_connectivity_state_destroy(&c->state_and_health_tracker); grpc_connector_unref(c->connector); grpc_pollset_set_destroy(c->pollset_set); - grpc_subchannel_key_destroy(c->key); + grpc_core::Delete(c->key); gpr_mu_destroy(&c->mu); gpr_free(c); } @@ -428,7 +431,12 @@ grpc_subchannel* grpc_subchannel_ref_from_weak_ref( } static void disconnect(grpc_subchannel* c) { - grpc_subchannel_index_unregister(c->key, c); + // The subchannel_pool is only used once here in this subchannel, so the + // access can be outside of the lock. + if (c->subchannel_pool != nullptr) { + c->subchannel_pool->UnregisterSubchannel(c->key); + c->subchannel_pool.reset(); + } gpr_mu_lock(&c->mu); GPR_ASSERT(!c->disconnected); c->disconnected = true; @@ -538,13 +546,17 @@ struct HealthCheckParams { grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, const grpc_channel_args* args) { - grpc_subchannel_key* key = grpc_subchannel_key_create(args); - grpc_subchannel* c = grpc_subchannel_index_find(key); - if (c) { - grpc_subchannel_key_destroy(key); + grpc_core::SubchannelKey* key = + grpc_core::New(args); + grpc_core::SubchannelPoolInterface* subchannel_pool = + grpc_core::SubchannelPoolInterface::GetSubchannelPoolFromChannelArgs( + args); + GPR_ASSERT(subchannel_pool != nullptr); + grpc_subchannel* c = subchannel_pool->FindSubchannel(key); + if (c != nullptr) { + grpc_core::Delete(key); return c; } - GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED(); c = static_cast(gpr_zalloc(sizeof(*c))); c->key = key; @@ -616,8 +628,13 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, grpc_core::channelz::ChannelTrace::Severity::Info, grpc_slice_from_static_string("Subchannel created")); } - - return grpc_subchannel_index_register(key, c); + // Try to register the subchannel before setting the subchannel pool. + // Otherwise, in case of a registration race, unreffing c in + // RegisterSubchannel() will cause c to be tried to be unregistered, while its + // key maps to a different subchannel. + grpc_subchannel* registered = subchannel_pool->RegisterSubchannel(key, c); + if (registered == c) c->subchannel_pool = subchannel_pool->Ref(); + return registered; } grpc_core::channelz::SubchannelNode* grpc_subchannel_get_channelz_node( @@ -983,11 +1000,6 @@ grpc_subchannel_get_connected_subchannel(grpc_subchannel* c) { return copy; } -const grpc_subchannel_key* grpc_subchannel_get_key( - const grpc_subchannel* subchannel) { - return subchannel->key; -} - void* grpc_connected_subchannel_call_get_parent_data( grpc_subchannel_call* subchannel_call) { grpc_channel_stack* chanstk = subchannel_call->connection->channel_stack(); diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 8c994c64f50..fac515eee5c 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -23,6 +23,7 @@ #include "src/core/ext/filters/client_channel/client_channel_channelz.h" #include "src/core/ext/filters/client_channel/connector.h" +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/gpr/arena.h" #include "src/core/lib/gprpp/ref_counted.h" @@ -38,7 +39,6 @@ address. Provides a target for load balancing. */ typedef struct grpc_subchannel grpc_subchannel; typedef struct grpc_subchannel_call grpc_subchannel_call; -typedef struct grpc_subchannel_key grpc_subchannel_key; #ifndef NDEBUG #define GRPC_SUBCHANNEL_REF(p, r) \ @@ -162,10 +162,6 @@ void grpc_subchannel_notify_on_state_change( grpc_core::RefCountedPtr grpc_subchannel_get_connected_subchannel(grpc_subchannel* c); -/** return the subchannel index key for \a subchannel */ -const grpc_subchannel_key* grpc_subchannel_get_key( - const grpc_subchannel* subchannel); - // Resets the connection backoff of the subchannel. // TODO(roth): Move connection backoff out of subchannels and up into LB // policy code (probably by adding a SubchannelGroup between diff --git a/src/core/ext/filters/client_channel/subchannel_pool_interface.cc b/src/core/ext/filters/client_channel/subchannel_pool_interface.cc new file mode 100644 index 00000000000..bb35f228b70 --- /dev/null +++ b/src/core/ext/filters/client_channel/subchannel_pool_interface.cc @@ -0,0 +1,97 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include + +#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h" + +#include "src/core/lib/gpr/useful.h" + +// The subchannel pool to reuse subchannels. +#define GRPC_ARG_SUBCHANNEL_POOL "grpc.subchannel_pool" +// The subchannel key ID that is only used in test to make each key unique. +#define GRPC_ARG_SUBCHANNEL_KEY_TEST_ONLY_ID "grpc.subchannel_key_test_only_id" + +namespace grpc_core { + +TraceFlag grpc_subchannel_pool_trace(false, "subchannel_pool"); + +SubchannelKey::SubchannelKey(const grpc_channel_args* args) { + Init(args, grpc_channel_args_normalize); +} + +SubchannelKey::~SubchannelKey() { + grpc_channel_args_destroy(const_cast(args_)); +} + +SubchannelKey::SubchannelKey(const SubchannelKey& other) { + Init(other.args_, grpc_channel_args_copy); +} + +SubchannelKey& SubchannelKey::operator=(const SubchannelKey& other) { + grpc_channel_args_destroy(const_cast(args_)); + Init(other.args_, grpc_channel_args_copy); + return *this; +} + +int SubchannelKey::Cmp(const SubchannelKey& other) const { + return grpc_channel_args_compare(args_, other.args_); +} + +void SubchannelKey::Init( + const grpc_channel_args* args, + grpc_channel_args* (*copy_channel_args)(const grpc_channel_args* args)) { + args_ = copy_channel_args(args); +} + +namespace { + +void* arg_copy(void* p) { + auto* subchannel_pool = static_cast(p); + subchannel_pool->Ref().release(); + return p; +} + +void arg_destroy(void* p) { + auto* subchannel_pool = static_cast(p); + subchannel_pool->Unref(); +} + +int arg_cmp(void* a, void* b) { return GPR_ICMP(a, b); } + +const grpc_arg_pointer_vtable subchannel_pool_arg_vtable = { + arg_copy, arg_destroy, arg_cmp}; + +} // namespace + +grpc_arg SubchannelPoolInterface::CreateChannelArg( + SubchannelPoolInterface* subchannel_pool) { + return grpc_channel_arg_pointer_create( + const_cast(GRPC_ARG_SUBCHANNEL_POOL), subchannel_pool, + &subchannel_pool_arg_vtable); +} + +SubchannelPoolInterface* +SubchannelPoolInterface::GetSubchannelPoolFromChannelArgs( + const grpc_channel_args* args) { + const grpc_arg* arg = grpc_channel_args_find(args, GRPC_ARG_SUBCHANNEL_POOL); + if (arg == nullptr || arg->type != GRPC_ARG_POINTER) return nullptr; + return static_cast(arg->value.pointer.p); +} + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/subchannel_pool_interface.h b/src/core/ext/filters/client_channel/subchannel_pool_interface.h new file mode 100644 index 00000000000..21597bf4276 --- /dev/null +++ b/src/core/ext/filters/client_channel/subchannel_pool_interface.h @@ -0,0 +1,94 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_SUBCHANNEL_POOL_INTERFACE_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_SUBCHANNEL_POOL_INTERFACE_H + +#include + +#include "src/core/lib/avl/avl.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/abstract.h" +#include "src/core/lib/gprpp/ref_counted.h" + +struct grpc_subchannel; + +namespace grpc_core { + +extern TraceFlag grpc_subchannel_pool_trace; + +// A key that can uniquely identify a subchannel. +class SubchannelKey { + public: + explicit SubchannelKey(const grpc_channel_args* args); + ~SubchannelKey(); + + // Copyable. + SubchannelKey(const SubchannelKey& other); + SubchannelKey& operator=(const SubchannelKey& other); + // Not movable. + SubchannelKey(SubchannelKey&&) = delete; + SubchannelKey& operator=(SubchannelKey&&) = delete; + + int Cmp(const SubchannelKey& other) const; + + private: + // Initializes the subchannel key with the given \a args and the function to + // copy channel args. + void Init( + const grpc_channel_args* args, + grpc_channel_args* (*copy_channel_args)(const grpc_channel_args* args)); + + const grpc_channel_args* args_; +}; + +// Interface for subchannel pool. +// TODO(juanlishen): This refcounting mechanism may lead to memory leak. +// To solve that, we should force polling to flush any pending callbacks, then +// shut down safely. See https://github.com/grpc/grpc/issues/12560. +class SubchannelPoolInterface : public RefCounted { + public: + SubchannelPoolInterface() : RefCounted(&grpc_subchannel_pool_trace) {} + virtual ~SubchannelPoolInterface() {} + + // Registers a subchannel against a key. Returns the subchannel registered + // with \a key, which may be different from \a constructed because we reuse + // (instead of update) any existing subchannel already registered with \a key. + virtual grpc_subchannel* RegisterSubchannel( + SubchannelKey* key, grpc_subchannel* constructed) GRPC_ABSTRACT; + + // Removes the registered subchannel found by \a key. + virtual void UnregisterSubchannel(SubchannelKey* key) GRPC_ABSTRACT; + + // Finds the subchannel registered for the given subchannel key. Returns NULL + // if no such channel exists. Thread-safe. + virtual grpc_subchannel* FindSubchannel(SubchannelKey* key) GRPC_ABSTRACT; + + // Creates a channel arg from \a subchannel pool. + static grpc_arg CreateChannelArg(SubchannelPoolInterface* subchannel_pool); + + // Gets the subchannel pool from the channel args. + static SubchannelPoolInterface* GetSubchannelPoolFromChannelArgs( + const grpc_channel_args* args); + + GRPC_ABSTRACT_BASE_CLASS +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_SUBCHANNEL_POOL_INTERFACE_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index f5e43ca657e..0272aae690d 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -319,11 +319,13 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/client_channel_factory.cc', 'src/core/ext/filters/client_channel/client_channel_plugin.cc', 'src/core/ext/filters/client_channel/connector.cc', + 'src/core/ext/filters/client_channel/global_subchannel_pool.cc', 'src/core/ext/filters/client_channel/health/health_check_client.cc', 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', + 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', 'src/core/ext/filters/client_channel/proxy_mapper.cc', 'src/core/ext/filters/client_channel/proxy_mapper_registry.cc', @@ -334,7 +336,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/retry_throttle.cc', 'src/core/ext/filters/client_channel/server_address.cc', 'src/core/ext/filters/client_channel/subchannel.cc', - 'src/core/ext/filters/client_channel/subchannel_index.cc', + 'src/core/ext/filters/client_channel/subchannel_pool_interface.cc', 'src/core/ext/filters/deadline/deadline_filter.cc', 'src/core/ext/filters/client_channel/health/health.pb.c', 'src/core/tsi/fake_transport_security.cc', diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 9783f51ab7d..aa8a6a96c4b 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,7 @@ #include #include +#include "src/core/ext/filters/client_channel/global_subchannel_pool.h" #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" #include "src/core/ext/filters/client_channel/server_address.h" @@ -96,6 +98,7 @@ class MyTestServiceImpl : public TestServiceImpl { std::unique_lock lock(mu_); ++request_count_; } + AddClient(context->peer()); return TestServiceImpl::Echo(context, request, response); } @@ -109,9 +112,21 @@ class MyTestServiceImpl : public TestServiceImpl { request_count_ = 0; } + std::set clients() { + std::unique_lock lock(clients_mu_); + return clients_; + } + private: + void AddClient(const grpc::string& client) { + std::unique_lock lock(clients_mu_); + clients_.insert(client); + } + std::mutex mu_; int request_count_; + std::mutex clients_mu_; + std::set clients_; }; class ClientLbEnd2endTest : public ::testing::Test { @@ -661,6 +676,54 @@ TEST_F(ClientLbEnd2endTest, PickFirstUpdateSuperset) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstGlobalSubchannelPool) { + // Start one server. + const int kNumServers = 1; + StartServers(kNumServers); + std::vector ports = GetServersPorts(); + // Create two channels that (by default) use the global subchannel pool. + auto channel1 = BuildChannel("pick_first"); + auto stub1 = BuildStub(channel1); + SetNextResolution(ports); + auto channel2 = BuildChannel("pick_first"); + auto stub2 = BuildStub(channel2); + SetNextResolution(ports); + WaitForServer(stub1, 0, DEBUG_LOCATION); + // Send one RPC on each channel. + CheckRpcSendOk(stub1, DEBUG_LOCATION); + CheckRpcSendOk(stub2, DEBUG_LOCATION); + // The server receives two requests. + EXPECT_EQ(2, servers_[0]->service_.request_count()); + // The two requests are from the same client port, because the two channels + // share subchannels via the global subchannel pool. + EXPECT_EQ(1UL, servers_[0]->service_.clients().size()); +} + +TEST_F(ClientLbEnd2endTest, PickFirstLocalSubchannelPool) { + // Start one server. + const int kNumServers = 1; + StartServers(kNumServers); + std::vector ports = GetServersPorts(); + // Create two channels that use local subchannel pool. + ChannelArguments args; + args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1); + auto channel1 = BuildChannel("pick_first", args); + auto stub1 = BuildStub(channel1); + SetNextResolution(ports); + auto channel2 = BuildChannel("pick_first", args); + auto stub2 = BuildStub(channel2); + SetNextResolution(ports); + WaitForServer(stub1, 0, DEBUG_LOCATION); + // Send one RPC on each channel. + CheckRpcSendOk(stub1, DEBUG_LOCATION); + CheckRpcSendOk(stub2, DEBUG_LOCATION); + // The server receives two requests. + EXPECT_EQ(2, servers_[0]->service_.request_count()); + // The two requests are from two client ports, because the two channels didn't + // share subchannels with each other. + EXPECT_EQ(2UL, servers_[0]->service_.clients().size()); +} + TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { const int kNumUpdates = 1000; const int kNumServers = 3; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index f739ed032bb..b589cd4044a 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -40,9 +40,8 @@ #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" -#include "src/cpp/server/secure_server_credentials.h" - #include "src/cpp/client/secure_credentials.h" +#include "src/cpp/server/secure_server_credentials.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index bb350550e94..51b9eda22b6 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -886,6 +886,8 @@ src/core/ext/filters/client_channel/client_channel_factory.h \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ src/core/ext/filters/client_channel/connector.h \ +src/core/ext/filters/client_channel/global_subchannel_pool.cc \ +src/core/ext/filters/client_channel/global_subchannel_pool.h \ src/core/ext/filters/client_channel/health/health.pb.c \ src/core/ext/filters/client_channel/health/health.pb.h \ src/core/ext/filters/client_channel/health/health_check_client.cc \ @@ -926,6 +928,8 @@ src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/lb_policy_registry.h \ +src/core/ext/filters/client_channel/local_subchannel_pool.cc \ +src/core/ext/filters/client_channel/local_subchannel_pool.h \ src/core/ext/filters/client_channel/parse_address.cc \ src/core/ext/filters/client_channel/parse_address.h \ src/core/ext/filters/client_channel/proxy_mapper.cc \ @@ -964,8 +968,8 @@ src/core/ext/filters/client_channel/server_address.cc \ src/core/ext/filters/client_channel/server_address.h \ src/core/ext/filters/client_channel/subchannel.cc \ src/core/ext/filters/client_channel/subchannel.h \ -src/core/ext/filters/client_channel/subchannel_index.cc \ -src/core/ext/filters/client_channel/subchannel_index.h \ +src/core/ext/filters/client_channel/subchannel_pool_interface.cc \ +src/core/ext/filters/client_channel/subchannel_pool_interface.h \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/deadline/deadline_filter.h \ src/core/ext/filters/http/client/http_client_filter.cc \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 197de64dbe1..8ab9c57142e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -9883,12 +9883,14 @@ "src/core/ext/filters/client_channel/client_channel_channelz.h", "src/core/ext/filters/client_channel/client_channel_factory.h", "src/core/ext/filters/client_channel/connector.h", + "src/core/ext/filters/client_channel/global_subchannel_pool.h", "src/core/ext/filters/client_channel/health/health_check_client.h", "src/core/ext/filters/client_channel/http_connect_handshaker.h", "src/core/ext/filters/client_channel/http_proxy.h", "src/core/ext/filters/client_channel/lb_policy.h", "src/core/ext/filters/client_channel/lb_policy_factory.h", "src/core/ext/filters/client_channel/lb_policy_registry.h", + "src/core/ext/filters/client_channel/local_subchannel_pool.h", "src/core/ext/filters/client_channel/parse_address.h", "src/core/ext/filters/client_channel/proxy_mapper.h", "src/core/ext/filters/client_channel/proxy_mapper_registry.h", @@ -9900,7 +9902,7 @@ "src/core/ext/filters/client_channel/retry_throttle.h", "src/core/ext/filters/client_channel/server_address.h", "src/core/ext/filters/client_channel/subchannel.h", - "src/core/ext/filters/client_channel/subchannel_index.h" + "src/core/ext/filters/client_channel/subchannel_pool_interface.h" ], "is_filegroup": true, "language": "c", @@ -9918,6 +9920,8 @@ "src/core/ext/filters/client_channel/client_channel_plugin.cc", "src/core/ext/filters/client_channel/connector.cc", "src/core/ext/filters/client_channel/connector.h", + "src/core/ext/filters/client_channel/global_subchannel_pool.cc", + "src/core/ext/filters/client_channel/global_subchannel_pool.h", "src/core/ext/filters/client_channel/health/health_check_client.cc", "src/core/ext/filters/client_channel/health/health_check_client.h", "src/core/ext/filters/client_channel/http_connect_handshaker.cc", @@ -9929,6 +9933,8 @@ "src/core/ext/filters/client_channel/lb_policy_factory.h", "src/core/ext/filters/client_channel/lb_policy_registry.cc", "src/core/ext/filters/client_channel/lb_policy_registry.h", + "src/core/ext/filters/client_channel/local_subchannel_pool.cc", + "src/core/ext/filters/client_channel/local_subchannel_pool.h", "src/core/ext/filters/client_channel/parse_address.cc", "src/core/ext/filters/client_channel/parse_address.h", "src/core/ext/filters/client_channel/proxy_mapper.cc", @@ -9950,8 +9956,8 @@ "src/core/ext/filters/client_channel/server_address.h", "src/core/ext/filters/client_channel/subchannel.cc", "src/core/ext/filters/client_channel/subchannel.h", - "src/core/ext/filters/client_channel/subchannel_index.cc", - "src/core/ext/filters/client_channel/subchannel_index.h" + "src/core/ext/filters/client_channel/subchannel_pool_interface.cc", + "src/core/ext/filters/client_channel/subchannel_pool_interface.h" ], "third_party": false, "type": "filegroup"