[grpclb and fake resolver] clean up e2e tests and simplify fake resolver (#34887)

Changes to fake resolver:
- Add `WaitForReresolutionRequest()` method to fake resolver response
generator to allow tests to tell when re-resolution has been requested.
- Change fake resolver response generator API to have only one mechanism
for injecting results, regardless of whether the result is an error or
whether it's triggered by a re-resolution.

Changes to grpclb_end2end_test:
- Change balancer interface such that instead of setting a list of
responses with fixed delays, the test can control exactly when each
response is set.
- Change balancer impl to always send the initial LB response, as
expected by the grpclb protocol.
- Change balancer impl to always read load reports, even if load
reporting is not expected to be enabled. (The latter case will still
cause the test to fail.) Reads are done in a different thread than
writes.
- Allow each test to directly control how many backends and balancers
are started and the client load reporting interval, so that (a) we don't
waste resources starting servers we don't need and (b) there is no need
to arbitrarily split tests across different test classes.
- Add timeouts to `WaitForAllBackends()` functionality, so that tests
will fail with a useful error rather than timing out.
- Improved ergonomics of various helper functions in the test framework.

In the process of making these changes, I found a couple of bugs:
- A bug in pick_first, which I fixed in #34885.
- A bug in grpclb, in which we were using the wrong condition to decide
whether to propagate a re-resolution request from the child policy,
which I've fixed in this PR. (This bug probably originated way back in
#18344.)

This should address a lot of the flakes seen in grpclb_e2e_test
recently.
pull/34890/head^2
Mark D. Roth 1 year ago committed by GitHub
parent 3869ef09a5
commit 8a000f45f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      BUILD
  2. 2
      src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc
  3. 33
      src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc
  4. 263
      src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc
  5. 71
      src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h
  6. 316
      test/core/client_channel/resolvers/fake_resolver_test.cc
  7. 9
      test/core/end2end/BUILD
  8. 25
      test/core/end2end/no_server_test.cc
  9. 46
      test/core/transport/chttp2/too_many_pings_test.cc
  10. 49
      test/cpp/end2end/client_lb_end2end_test.cc
  11. 1701
      test/cpp/end2end/grpclb_end2end_test.cc

@ -3748,9 +3748,9 @@ grpc_cc_library(
hdrs = ["//src/core:ext/filters/client_channel/resolver/fake/fake_resolver.h"],
external_deps = [
"absl/base:core_headers",
"absl/status",
"absl/status:statusor",
"absl/strings",
"absl/time",
"absl/types:optional",
],
language = "c++",
visibility = [
@ -3760,7 +3760,6 @@ grpc_cc_library(
deps = [
"config",
"debug_location",
"endpoint_addresses",
"gpr",
"grpc_public_hdrs",
"grpc_resolver",
@ -3769,7 +3768,6 @@ grpc_cc_library(
"uri_parser",
"work_serializer",
"//src/core:channel_args",
"//src/core:grpc_service_config",
"//src/core:notification",
"//src/core:ref_counted",
"//src/core:useful",

@ -98,7 +98,7 @@ class ChildPolicyHandler::Helper
: parent()->child_policy_.get();
if (child_ != latest_child_policy) return;
if (GRPC_TRACE_FLAG_ENABLED(*(parent()->tracer_))) {
gpr_log(GPR_INFO, "[child_policy_handler %p] started name re-resolving",
gpr_log(GPR_INFO, "[child_policy_handler %p] requesting re-resolution",
parent());
}
parent()->channel_control_helper()->RequestReresolution();

@ -841,14 +841,10 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
void GrpcLb::Helper::RequestReresolution() {
if (parent()->shutting_down_) return;
// If we are talking to a balancer, we expect to get updated addresses
// from the balancer, so we can ignore the re-resolution request from
// the child policy. Otherwise, pass the re-resolution request up to the
// channel.
if (parent()->lb_calld_ == nullptr ||
!parent()->lb_calld_->seen_initial_response()) {
parent()->channel_control_helper()->RequestReresolution();
}
// Ignore if we're not in fallback mode, because if we got the backend
// addresses from the balancer, re-resolving is not going to fix it.
if (!parent()->fallback_mode_) return;
parent()->channel_control_helper()->RequestReresolution();
}
//
@ -1508,6 +1504,9 @@ void GrpcLb::ResetBackoffLocked() {
}
absl::Status GrpcLb::UpdateLocked(UpdateArgs args) {
if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "[grpclb %p] received update", this);
}
const bool is_initial_update = lb_channel_ == nullptr;
config_ = args.config;
GPR_ASSERT(config_ != nullptr);
@ -1516,11 +1515,15 @@ absl::Status GrpcLb::UpdateLocked(UpdateArgs args) {
fallback_backend_addresses_ = std::move(args.addresses);
if (fallback_backend_addresses_.ok()) {
// Add null LB token attributes.
for (EndpointAddresses& addresses : *fallback_backend_addresses_) {
addresses = EndpointAddresses(
addresses.addresses(),
addresses.args().SetObject(
for (EndpointAddresses& endpoint : *fallback_backend_addresses_) {
endpoint = EndpointAddresses(
endpoint.addresses(),
endpoint.args().SetObject(
MakeRefCounted<TokenAndClientStatsArg>("", nullptr)));
if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "[grpclb %p] fallback address: %s", this,
endpoint.ToString().c_str());
}
}
}
resolution_note_ = std::move(args.resolution_note);
@ -1569,6 +1572,12 @@ absl::Status GrpcLb::UpdateLocked(UpdateArgs args) {
absl::Status GrpcLb::UpdateBalancerChannelLocked() {
// Get balancer addresses.
EndpointAddressesList balancer_addresses = ExtractBalancerAddresses(args_);
if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
for (const auto& endpoint : balancer_addresses) {
gpr_log(GPR_INFO, "[grpclb %p] balancer address: %s", this,
endpoint.ToString().c_str());
}
}
absl::Status status;
if (balancer_addresses.empty()) {
status = absl::UnavailableError("balancer address list must be non-empty");

@ -22,10 +22,9 @@
#include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
#include <memory>
#include <type_traits>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include <grpc/support/log.h>
@ -36,9 +35,7 @@
#include "src/core/lib/gprpp/debug_location.h"
#include "src/core/lib/gprpp/orphanable.h"
#include "src/core/lib/gprpp/work_serializer.h"
#include "src/core/lib/resolver/endpoint_addresses.h"
#include "src/core/lib/resolver/resolver_factory.h"
#include "src/core/lib/service_config/service_config.h"
#include "src/core/lib/uri/uri_parser.h"
namespace grpc_core {
@ -55,47 +52,38 @@ class FakeResolver : public Resolver {
private:
friend class FakeResolverResponseGenerator;
friend class FakeResolverResponseSetter;
void ShutdownLocked() override;
void MaybeSendResultLocked();
void ReturnReresolutionResult();
// passed-in parameters
ChannelArgs channel_args_;
std::shared_ptr<WorkSerializer> work_serializer_;
std::unique_ptr<ResultHandler> result_handler_;
ChannelArgs channel_args_;
RefCountedPtr<FakeResolverResponseGenerator> response_generator_;
// If has_next_result_ is true, next_result_ is the next resolution result
// to be returned.
bool has_next_result_ = false;
Result next_result_;
// Result to use for the pretended re-resolution in
// RequestReresolutionLocked().
bool has_reresolution_result_ = false;
Result reresolution_result_;
// The next resolution result to be returned, if any. Present when we
// get a result before the resolver is started.
absl::optional<Result> next_result_;
// True after the call to StartLocked().
bool started_ = false;
// True after the call to ShutdownLocked().
bool shutdown_ = false;
// if true, return failure
bool return_failure_ = false;
// pending re-resolution
bool reresolution_closure_pending_ = false;
};
FakeResolver::FakeResolver(ResolverArgs args)
: work_serializer_(std::move(args.work_serializer)),
result_handler_(std::move(args.result_handler)),
channel_args_(
// Channels sharing the same subchannels may have different resolver
// response generators. If we don't remove this arg, subchannel pool
// will create new subchannels for the same address instead of
// reusing existing ones because of different values of this channel
// arg. Can't just use GRPC_ARG_NO_SUBCHANNEL_PREFIX, since
// that can't be passed into the channel from test code.
args.args.Remove(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR)),
response_generator_(
args.args.GetObjectRef<FakeResolverResponseGenerator>()) {
// Channels sharing the same subchannels may have different resolver response
// generators. If we don't remove this arg, subchannel pool will create new
// subchannels for the same address instead of reusing existing ones because
// of different values of this channel arg.
channel_args_ = args.args.Remove(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR);
if (response_generator_ != nullptr) {
response_generator_->SetFakeResolver(Ref());
}
@ -107,19 +95,9 @@ void FakeResolver::StartLocked() {
}
void FakeResolver::RequestReresolutionLocked() {
if (has_reresolution_result_ || return_failure_) {
next_result_ = reresolution_result_;
has_next_result_ = true;
// Return the result in a different closure, so that we don't call
// back into the LB policy while it's still processing the previous
// update.
if (!reresolution_closure_pending_) {
reresolution_closure_pending_ = true;
Ref().release(); // ref held by closure
work_serializer_->Run([this]() { ReturnReresolutionResult(); },
DEBUG_LOCATION);
}
}
// Re-resolution can't happen until after we return an initial result.
GPR_ASSERT(response_generator_ != nullptr);
response_generator_->ReresolutionRequested();
}
void FakeResolver::ShutdownLocked() {
@ -132,80 +110,15 @@ void FakeResolver::ShutdownLocked() {
void FakeResolver::MaybeSendResultLocked() {
if (!started_ || shutdown_) return;
if (return_failure_) {
// TODO(roth): Change resolver result generator to be able to inject
// the error to be returned and to be able to independently set errors
// for addresses and service config.
Result result;
result.addresses = absl::UnavailableError("Resolver transient failure");
result.service_config = result.addresses.status();
result.args = channel_args_;
result_handler_->ReportResult(std::move(result));
return_failure_ = false;
} else if (has_next_result_) {
if (next_result_.has_value()) {
// When both next_results_ and channel_args_ contain an arg with the same
// name, only the one in next_results_.
next_result_.args = next_result_.args.UnionWith(channel_args_);
result_handler_->ReportResult(std::move(next_result_));
has_next_result_ = false;
// name, use the one in next_results_.
next_result_->args = next_result_->args.UnionWith(channel_args_);
result_handler_->ReportResult(std::move(*next_result_));
next_result_.reset();
}
}
void FakeResolver::ReturnReresolutionResult() {
reresolution_closure_pending_ = false;
MaybeSendResultLocked();
Unref();
}
class FakeResolverResponseSetter {
public:
explicit FakeResolverResponseSetter(RefCountedPtr<FakeResolver> resolver,
Resolver::Result result,
bool has_result = false,
bool immediate = true)
: resolver_(std::move(resolver)),
result_(std::move(result)),
has_result_(has_result),
immediate_(immediate) {}
void SetResponseLocked();
void SetReresolutionResponseLocked();
void SetFailureLocked();
private:
RefCountedPtr<FakeResolver> resolver_;
Resolver::Result result_;
bool has_result_;
bool immediate_;
};
// Deletes object when done
void FakeResolverResponseSetter::SetReresolutionResponseLocked() {
if (!resolver_->shutdown_) {
resolver_->reresolution_result_ = std::move(result_);
resolver_->has_reresolution_result_ = has_result_;
}
delete this;
}
// Deletes object when done
void FakeResolverResponseSetter::SetResponseLocked() {
if (!resolver_->shutdown_) {
resolver_->next_result_ = std::move(result_);
resolver_->has_next_result_ = true;
resolver_->MaybeSendResultLocked();
}
delete this;
}
// Deletes object when done
void FakeResolverResponseSetter::SetFailureLocked() {
if (!resolver_->shutdown_) {
resolver_->return_failure_ = true;
if (immediate_) resolver_->MaybeSendResultLocked();
}
delete this;
}
//
// FakeResolverResponseGenerator
//
@ -220,101 +133,73 @@ void FakeResolverResponseGenerator::SetResponseAndNotify(
{
MutexLock lock(&mu_);
if (resolver_ == nullptr) {
has_result_ = true;
result_ = std::move(result);
if (notify_when_set != nullptr) notify_when_set->Notify();
return;
}
resolver = resolver_->Ref();
}
FakeResolverResponseSetter* arg =
new FakeResolverResponseSetter(resolver, std::move(result));
resolver->work_serializer_->Run(
[arg, notify_when_set]() {
arg->SetResponseLocked();
if (notify_when_set != nullptr) notify_when_set->Notify();
},
DEBUG_LOCATION);
SendResultToResolver(std::move(resolver), std::move(result), notify_when_set);
}
void FakeResolverResponseGenerator::SetReresolutionResponseAndNotify(
Resolver::Result result, Notification* notify_when_set) {
RefCountedPtr<FakeResolver> resolver;
void FakeResolverResponseGenerator::SetFakeResolver(
RefCountedPtr<FakeResolver> resolver) {
Resolver::Result result;
{
MutexLock lock(&mu_);
GPR_ASSERT(resolver_ != nullptr);
resolver = resolver_->Ref();
resolver_ = resolver;
if (resolver_set_cv_ != nullptr) resolver_set_cv_->SignalAll();
if (resolver == nullptr) return;
if (!result_.has_value()) return;
result = std::move(*result_);
result_.reset();
}
FakeResolverResponseSetter* arg = new FakeResolverResponseSetter(
resolver, std::move(result), true /* has_result */);
resolver->work_serializer_->Run(
[arg, notify_when_set]() {
arg->SetReresolutionResponseLocked();
SendResultToResolver(std::move(resolver), std::move(result), nullptr);
}
void FakeResolverResponseGenerator::SendResultToResolver(
RefCountedPtr<FakeResolver> resolver, Resolver::Result result,
Notification* notify_when_set) {
auto* resolver_ptr = resolver.get();
resolver_ptr->work_serializer_->Run(
[resolver = std::move(resolver), result = std::move(result),
notify_when_set]() mutable {
if (!resolver->shutdown_) {
resolver->next_result_ = std::move(result);
resolver->MaybeSendResultLocked();
}
if (notify_when_set != nullptr) notify_when_set->Notify();
},
DEBUG_LOCATION);
}
void FakeResolverResponseGenerator::UnsetReresolutionResponse() {
RefCountedPtr<FakeResolver> resolver;
{
MutexLock lock(&mu_);
GPR_ASSERT(resolver_ != nullptr);
resolver = resolver_->Ref();
}
FakeResolverResponseSetter* arg =
new FakeResolverResponseSetter(resolver, Resolver::Result());
resolver->work_serializer_->Run(
[arg]() { arg->SetReresolutionResponseLocked(); }, DEBUG_LOCATION);
}
void FakeResolverResponseGenerator::SetFailure() {
RefCountedPtr<FakeResolver> resolver;
{
MutexLock lock(&mu_);
GPR_ASSERT(resolver_ != nullptr);
resolver = resolver_->Ref();
}
FakeResolverResponseSetter* arg =
new FakeResolverResponseSetter(resolver, Resolver::Result());
resolver->work_serializer_->Run([arg]() { arg->SetFailureLocked(); },
DEBUG_LOCATION);
}
void FakeResolverResponseGenerator::SetFailureOnReresolution() {
RefCountedPtr<FakeResolver> resolver;
{
MutexLock lock(&mu_);
GPR_ASSERT(resolver_ != nullptr);
resolver = resolver_->Ref();
bool FakeResolverResponseGenerator::WaitForResolverSet(absl::Duration timeout) {
MutexLock lock(&mu_);
if (resolver_ == nullptr) {
CondVar condition;
resolver_set_cv_ = &condition;
condition.WaitWithTimeout(&mu_, timeout);
resolver_set_cv_ = nullptr;
}
FakeResolverResponseSetter* arg = new FakeResolverResponseSetter(
resolver, Resolver::Result(), false /* has_result */,
false /* immediate */);
resolver->work_serializer_->Run([arg]() { arg->SetFailureLocked(); },
DEBUG_LOCATION);
return resolver_ != nullptr;
}
void FakeResolverResponseGenerator::SetFakeResolver(
RefCountedPtr<FakeResolver> resolver) {
MutexLock lock(&mu_);
resolver_ = std::move(resolver);
cv_.SignalAll();
if (resolver_ == nullptr) return;
if (has_result_) {
FakeResolverResponseSetter* arg =
new FakeResolverResponseSetter(resolver_, std::move(result_));
resolver_->work_serializer_->Run([arg]() { arg->SetResponseLocked(); },
DEBUG_LOCATION);
has_result_ = false;
bool FakeResolverResponseGenerator::WaitForReresolutionRequest(
absl::Duration timeout) {
MutexLock lock(&reresolution_mu_);
if (!reresolution_requested_) {
CondVar condition;
reresolution_cv_ = &condition;
condition.WaitWithTimeout(&reresolution_mu_, timeout);
reresolution_cv_ = nullptr;
}
return std::exchange(reresolution_requested_, false);
}
void FakeResolverResponseGenerator::WaitForResolverSet() {
MutexLock lock(&mu_);
while (resolver_ == nullptr) {
cv_.Wait(&mu_);
}
void FakeResolverResponseGenerator::ReresolutionRequested() {
MutexLock lock(&reresolution_mu_);
reresolution_requested_ = true;
if (reresolution_cv_ != nullptr) reresolution_cv_->SignalAll();
}
namespace {
@ -341,22 +226,6 @@ const grpc_arg_pointer_vtable
ResponseGeneratorChannelArgCopy, ResponseGeneratorChannelArgDestroy,
ResponseGeneratorChannelArgCmp};
grpc_arg FakeResolverResponseGenerator::MakeChannelArg(
FakeResolverResponseGenerator* generator) {
return grpc_channel_arg_pointer_create(
const_cast<char*>(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR), generator,
&kChannelArgPointerVtable);
}
RefCountedPtr<FakeResolverResponseGenerator>
FakeResolverResponseGenerator::GetFromArgs(const grpc_channel_args* args) {
auto* response_generator =
grpc_channel_args_find_pointer<FakeResolverResponseGenerator>(
args, GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR);
if (response_generator == nullptr) return nullptr;
return response_generator->Ref();
}
//
// Factory
//

@ -23,6 +23,8 @@
#include "absl/base/thread_annotations.h"
#include "absl/strings/string_view.h"
#include "absl/time/time.h"
#include "absl/types/optional.h"
#include <grpc/grpc.h>
@ -42,8 +44,7 @@ class FakeResolver;
/// A mechanism for generating responses for the fake resolver.
/// An instance of this class is passed to the fake resolver via a channel
/// argument (see \a MakeChannelArg()) and used to inject and trigger custom
/// resolutions.
/// argument and used to inject and trigger custom resolutions.
// TODO(roth): I would ideally like this to be InternallyRefCounted
// instead of RefCounted, but external refs are currently needed to
// encode this in channel args. Once channel_args are converted to C++,
@ -77,50 +78,20 @@ class FakeResolverResponseGenerator
n.WaitForNotification();
}
// Sets the re-resolution response, which is returned by the fake resolver
// when re-resolution is requested (via \a RequestReresolutionLocked()).
// The new re-resolution response replaces any previous re-resolution
// response that may have been set by a previous call.
// notify_when_set is an optional notification to signal when the response has
// been set.
void SetReresolutionResponseAndNotify(Resolver::Result result,
Notification* notify_when_set);
void SetReresolutionResponseAsync(Resolver::Result result) {
SetReresolutionResponseAndNotify(std::move(result), nullptr);
}
void SetReresolutionResponseSynchronously(Resolver::Result result) {
Notification n;
SetReresolutionResponseAndNotify(std::move(result), &n);
n.WaitForNotification();
}
// Unsets the re-resolution response. After this, the fake resolver will
// not return anything when \a RequestReresolutionLocked() is called.
void UnsetReresolutionResponse();
// Tells the resolver to return a transient failure.
void SetFailure();
// Same as SetFailure(), but instead of returning the error
// immediately, waits for the next call to RequestReresolutionLocked().
void SetFailureOnReresolution();
// Returns a channel arg containing \a generator.
// TODO(roth): When we have time, make this a non-static method.
static grpc_arg MakeChannelArg(FakeResolverResponseGenerator* generator);
// Waits up to timeout for a re-resolution request. Returns true if a
// re-resolution request is seen, or false if timeout occurs. Returns
// true immediately if there was a re-resolution request since the
// last time this method was called.
bool WaitForReresolutionRequest(absl::Duration timeout);
// Returns the response generator in \a args, or null if not found.
static RefCountedPtr<FakeResolverResponseGenerator> GetFromArgs(
const grpc_channel_args* args);
// Wait for a resolver to be set (setting may be happening asynchronously, so
// this may block - consider it test only).
bool WaitForResolverSet(absl::Duration timeout);
static absl::string_view ChannelArgName() {
return GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR;
}
// Wait for a resolver to be set (setting may be happening asynchronously, so
// this may block - consider it test only).
void WaitForResolverSet();
static int ChannelArgsCompare(const FakeResolverResponseGenerator* a,
const FakeResolverResponseGenerator* b) {
return QsortCompare(a, b);
@ -128,15 +99,29 @@ class FakeResolverResponseGenerator
private:
friend class FakeResolver;
// Set the corresponding FakeResolver to this generator.
void SetFakeResolver(RefCountedPtr<FakeResolver> resolver);
// Called by FakeResolver when re-resolution is requested.
void ReresolutionRequested();
// Helper function to send a result to the resolver.
static void SendResultToResolver(RefCountedPtr<FakeResolver> resolver,
Resolver::Result result,
Notification* notify_when_set);
// Mutex protecting the members below.
Mutex mu_;
CondVar cv_;
CondVar* resolver_set_cv_ ABSL_GUARDED_BY(mu_) = nullptr;
RefCountedPtr<FakeResolver> resolver_ ABSL_GUARDED_BY(mu_);
Resolver::Result result_ ABSL_GUARDED_BY(mu_);
bool has_result_ ABSL_GUARDED_BY(mu_) = false;
// Temporarily stores the result when it gets set before the response
// generator is seen by the FakeResolver.
absl::optional<Resolver::Result> result_ ABSL_GUARDED_BY(mu_);
Mutex reresolution_mu_;
CondVar* reresolution_cv_ ABSL_GUARDED_BY(reresolution_mu_) = nullptr;
bool reresolution_requested_ ABSL_GUARDED_BY(reresolution_mu_) = false;
};
} // namespace grpc_core

@ -32,11 +32,10 @@
#include "absl/container/inlined_vector.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/synchronization/notification.h"
#include "gtest/gtest.h"
#include <grpc/grpc.h>
#include <grpc/support/log.h>
#include <grpc/support/sync.h>
#include "src/core/lib/address_utils/parse_address.h"
#include "src/core/lib/channel/channel_args.h"
@ -54,180 +53,175 @@
#include "src/core/lib/uri/uri_parser.h"
#include "test/core/util/test_config.h"
class ResultHandler : public grpc_core::Resolver::ResultHandler {
public:
void SetExpectedAndEvent(grpc_core::Resolver::Result expected,
gpr_event* ev) {
grpc_core::MutexLock lock(&mu_);
ASSERT_EQ(ev_, nullptr);
expected_ = std::move(expected);
ev_ = ev;
}
namespace grpc_core {
namespace testing {
void ReportResult(grpc_core::Resolver::Result actual) override {
grpc_core::MutexLock lock(&mu_);
ASSERT_NE(ev_, nullptr);
// We only check the addresses, because that's the only thing
// explicitly set by the test via
// FakeResolverResponseGenerator::SetResponse().
ASSERT_TRUE(actual.addresses.ok());
ASSERT_EQ(actual.addresses->size(), expected_.addresses->size());
for (size_t i = 0; i < expected_.addresses->size(); ++i) {
ASSERT_EQ((*actual.addresses)[i], (*expected_.addresses)[i]);
class FakeResolverTest : public ::testing::Test {
protected:
class ResultHandler : public Resolver::ResultHandler {
public:
void SetExpectedAndNotification(Resolver::Result expected,
absl::Notification* notification) {
MutexLock lock(&mu_);
ASSERT_EQ(notification_, nullptr);
expected_ = std::move(expected);
notification_ = notification;
}
gpr_event_set(ev_, reinterpret_cast<void*>(1));
ev_ = nullptr;
}
private:
grpc_core::Mutex mu_;
grpc_core::Resolver::Result expected_ ABSL_GUARDED_BY(mu_);
gpr_event* ev_ ABSL_GUARDED_BY(mu_) = nullptr;
};
void ReportResult(Resolver::Result actual) override {
MutexLock lock(&mu_);
ASSERT_NE(notification_, nullptr);
// TODO(roth): Check fields other than just the addresses.
// Note: No good way to compare result_health_callback.
ASSERT_TRUE(actual.addresses.ok());
ASSERT_EQ(actual.addresses->size(), expected_.addresses->size());
for (size_t i = 0; i < expected_.addresses->size(); ++i) {
ASSERT_EQ((*actual.addresses)[i], (*expected_.addresses)[i]);
}
notification_->Notify();
notification_ = nullptr;
}
static grpc_core::OrphanablePtr<grpc_core::Resolver> build_fake_resolver(
std::shared_ptr<grpc_core::WorkSerializer> work_serializer,
grpc_core::FakeResolverResponseGenerator* response_generator,
std::unique_ptr<grpc_core::Resolver::ResultHandler> result_handler) {
grpc_core::ResolverFactory* factory = grpc_core::CoreConfiguration::Get()
.resolver_registry()
.LookupResolverFactory("fake");
grpc_arg generator_arg =
grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
response_generator);
grpc_channel_args channel_args = {1, &generator_arg};
grpc_core::ResolverArgs args;
args.args = grpc_core::ChannelArgs::FromC(&channel_args);
args.work_serializer = std::move(work_serializer);
args.result_handler = std::move(result_handler);
grpc_core::OrphanablePtr<grpc_core::Resolver> resolver =
factory->CreateResolver(std::move(args));
return resolver;
}
private:
Mutex mu_;
Resolver::Result expected_ ABSL_GUARDED_BY(mu_);
absl::Notification* notification_ ABSL_GUARDED_BY(mu_) = nullptr;
};
// Create a new resolution containing 2 addresses.
static grpc_core::Resolver::Result create_new_resolver_result() {
static size_t test_counter = 0;
const size_t num_addresses = 2;
// Create address list.
grpc_core::EndpointAddressesList addresses;
for (size_t i = 0; i < num_addresses; ++i) {
std::string uri_string = absl::StrFormat("ipv4:127.0.0.1:100%" PRIuPTR,
test_counter * num_addresses + i);
absl::StatusOr<grpc_core::URI> uri = grpc_core::URI::Parse(uri_string);
EXPECT_TRUE(uri.ok());
grpc_resolved_address address;
EXPECT_TRUE(grpc_parse_uri(*uri, &address));
absl::InlinedVector<grpc_arg, 2> args_to_add;
addresses.emplace_back(address, grpc_core::ChannelArgs());
static OrphanablePtr<Resolver> BuildFakeResolver(
std::shared_ptr<WorkSerializer> work_serializer,
RefCountedPtr<FakeResolverResponseGenerator> response_generator,
std::unique_ptr<Resolver::ResultHandler> result_handler) {
ResolverFactory* factory =
CoreConfiguration::Get().resolver_registry().LookupResolverFactory(
"fake");
ResolverArgs args;
args.args = ChannelArgs().SetObject(std::move(response_generator));
args.work_serializer = std::move(work_serializer);
args.result_handler = std::move(result_handler);
return factory->CreateResolver(std::move(args));
}
++test_counter;
grpc_core::Resolver::Result result;
result.addresses = std::move(addresses);
return result;
}
TEST(FakeResolverTest, FakeResolver) {
grpc_core::ExecCtx exec_ctx;
std::shared_ptr<grpc_core::WorkSerializer> work_serializer =
std::make_shared<grpc_core::WorkSerializer>(
grpc_event_engine::experimental::GetDefaultEventEngine());
auto synchronously = [work_serializer](std::function<void()> do_this_thing) {
grpc_core::Notification notification;
work_serializer->Run(
[do_this_thing = std::move(do_this_thing), &notification]() mutable {
do_this_thing();
// Create a new resolution containing 2 addresses.
static Resolver::Result CreateResolverResult() {
static size_t test_counter = 0;
const size_t num_addresses = 2;
// Create address list.
EndpointAddressesList addresses;
for (size_t i = 0; i < num_addresses; ++i) {
std::string uri_string = absl::StrFormat(
"ipv4:127.0.0.1:100%" PRIuPTR, test_counter * num_addresses + i);
absl::StatusOr<URI> uri = URI::Parse(uri_string);
EXPECT_TRUE(uri.ok());
grpc_resolved_address address;
EXPECT_TRUE(grpc_parse_uri(*uri, &address));
absl::InlinedVector<grpc_arg, 2> args_to_add;
addresses.emplace_back(address, ChannelArgs());
}
++test_counter;
Resolver::Result result;
result.addresses = std::move(addresses);
return result;
}
OrphanablePtr<Resolver> CreateResolver() {
result_handler_ = new ResultHandler();
return BuildFakeResolver(
work_serializer_, response_generator_,
std::unique_ptr<Resolver::ResultHandler>(result_handler_));
}
void RunSynchronously(std::function<void()> callback) {
Notification notification;
work_serializer_->Run(
[callback = std::move(callback), &notification]() {
callback();
notification.Notify();
},
DEBUG_LOCATION);
notification.WaitForNotification();
};
}
ExecCtx exec_ctx_;
std::shared_ptr<WorkSerializer> work_serializer_ =
std::make_shared<WorkSerializer>(
grpc_event_engine::experimental::GetDefaultEventEngine());
RefCountedPtr<FakeResolverResponseGenerator> response_generator_ =
MakeRefCounted<FakeResolverResponseGenerator>();
ResultHandler* result_handler_ = nullptr;
};
TEST_F(FakeResolverTest, WaitForResolverSet) {
EXPECT_FALSE(response_generator_->WaitForResolverSet(absl::Milliseconds(1)));
auto resolver = CreateResolver();
ASSERT_NE(resolver, nullptr);
EXPECT_TRUE(response_generator_->WaitForResolverSet(absl::Milliseconds(1)));
}
TEST_F(FakeResolverTest, ReturnResultBeforeResolverCreated) {
// Return result via response generator.
Resolver::Result result = CreateResolverResult();
response_generator_->SetResponseAsync(result);
// Create and start resolver.
auto resolver = CreateResolver();
ASSERT_NE(resolver, nullptr);
absl::Notification notification;
result_handler_->SetExpectedAndNotification(std::move(result), &notification);
RunSynchronously([resolver = resolver.get()] { resolver->StartLocked(); });
// Expect result.
ASSERT_TRUE(notification.WaitForNotificationWithTimeout(
absl::Seconds(5 * grpc_test_slowdown_factor())));
}
TEST_F(FakeResolverTest, ReturnResultBeforeResolverStarted) {
// Create resolver.
ResultHandler* result_handler = new ResultHandler();
grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
response_generator =
grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
grpc_core::OrphanablePtr<grpc_core::Resolver> resolver = build_fake_resolver(
work_serializer, response_generator.get(),
std::unique_ptr<grpc_core::Resolver::ResultHandler>(result_handler));
ASSERT_NE(resolver.get(), nullptr);
synchronously([resolver = resolver.get()] { resolver->StartLocked(); });
// Test 1: normal resolution.
// next_results != NULL, reresolution_results == NULL.
// Expected response is next_results.
gpr_log(GPR_INFO, "TEST 1");
grpc_core::Resolver::Result result = create_new_resolver_result();
gpr_event ev1;
gpr_event_init(&ev1);
result_handler->SetExpectedAndEvent(result, &ev1);
response_generator->SetResponseSynchronously(std::move(result));
grpc_core::ExecCtx::Get()->Flush();
ASSERT_NE(gpr_event_wait(&ev1, grpc_timeout_seconds_to_deadline(5)), nullptr);
// Test 2: update resolution.
// next_results != NULL, reresolution_results == NULL.
// Expected response is next_results.
gpr_log(GPR_INFO, "TEST 2");
result = create_new_resolver_result();
gpr_event ev2;
gpr_event_init(&ev2);
result_handler->SetExpectedAndEvent(result, &ev2);
response_generator->SetResponseSynchronously(std::move(result));
grpc_core::ExecCtx::Get()->Flush();
ASSERT_NE(gpr_event_wait(&ev2, grpc_timeout_seconds_to_deadline(5)), nullptr);
// Test 3: normal re-resolution.
// next_results == NULL, reresolution_results != NULL.
// Expected response is reresolution_results.
gpr_log(GPR_INFO, "TEST 3");
grpc_core::Resolver::Result reresolution_result =
create_new_resolver_result();
gpr_event ev3;
gpr_event_init(&ev3);
result_handler->SetExpectedAndEvent(reresolution_result, &ev3);
// Set reresolution_results.
// No result will be returned until re-resolution is requested.
response_generator->SetReresolutionResponseSynchronously(reresolution_result);
grpc_core::ExecCtx::Get()->Flush();
// Trigger a re-resolution.
synchronously(
[resolver = resolver.get()] { resolver->RequestReresolutionLocked(); });
grpc_core::ExecCtx::Get()->Flush();
ASSERT_NE(gpr_event_wait(&ev3, grpc_timeout_seconds_to_deadline(5)), nullptr);
// Test 4: repeat re-resolution.
// next_results == NULL, reresolution_results != NULL.
// Expected response is reresolution_results.
gpr_log(GPR_INFO, "TEST 4");
gpr_event ev4;
gpr_event_init(&ev4);
result_handler->SetExpectedAndEvent(std::move(reresolution_result), &ev4);
// Trigger a re-resolution.
synchronously(
auto resolver = CreateResolver();
ASSERT_NE(resolver, nullptr);
Resolver::Result result = CreateResolverResult();
absl::Notification notification;
result_handler_->SetExpectedAndNotification(result, &notification);
// Return result via response generator.
response_generator_->SetResponseAsync(std::move(result));
// Start resolver.
RunSynchronously([resolver = resolver.get()] { resolver->StartLocked(); });
// Expect result.
ASSERT_TRUE(notification.WaitForNotificationWithTimeout(
absl::Seconds(5 * grpc_test_slowdown_factor())));
}
TEST_F(FakeResolverTest, ReturnResult) {
// Create and start resolver.
auto resolver = CreateResolver();
ASSERT_NE(resolver, nullptr);
RunSynchronously([resolver = resolver.get()] { resolver->StartLocked(); });
Resolver::Result result = CreateResolverResult();
absl::Notification notification;
result_handler_->SetExpectedAndNotification(result, &notification);
// Return result via response generator.
response_generator_->SetResponseAsync(std::move(result));
// Expect result.
ASSERT_TRUE(notification.WaitForNotificationWithTimeout(
absl::Seconds(5 * grpc_test_slowdown_factor())));
}
TEST_F(FakeResolverTest, WaitForReresolutionRequest) {
// Create and start resolver.
auto resolver = CreateResolver();
ASSERT_NE(resolver, nullptr);
RunSynchronously([resolver = resolver.get()] { resolver->StartLocked(); });
// No re-resolution requested yet.
EXPECT_FALSE(
response_generator_->WaitForReresolutionRequest(absl::Milliseconds(1)));
// Request re-resolution, then try again.
RunSynchronously(
[resolver = resolver.get()] { resolver->RequestReresolutionLocked(); });
grpc_core::ExecCtx::Get()->Flush();
ASSERT_NE(gpr_event_wait(&ev4, grpc_timeout_seconds_to_deadline(5)), nullptr);
// Test 5: normal resolution.
// next_results != NULL, reresolution_results != NULL.
// Expected response is next_results.
gpr_log(GPR_INFO, "TEST 5");
result = create_new_resolver_result();
gpr_event ev5;
gpr_event_init(&ev5);
result_handler->SetExpectedAndEvent(result, &ev5);
response_generator->SetResponseSynchronously(std::move(result));
grpc_core::ExecCtx::Get()->Flush();
ASSERT_NE(gpr_event_wait(&ev5, grpc_timeout_seconds_to_deadline(5)), nullptr);
// Test 6: no-op.
// Requesting a new resolution without setting the response shouldn't trigger
// the resolution callback.
gpr_log(GPR_INFO, "TEST 6");
gpr_event ev6;
gpr_event_init(&ev6);
result_handler->SetExpectedAndEvent(grpc_core::Resolver::Result(), &ev6);
ASSERT_EQ(gpr_event_wait(&ev6, grpc_timeout_milliseconds_to_deadline(100)),
nullptr);
// Clean up.
resolver.reset();
EXPECT_TRUE(
response_generator_->WaitForReresolutionRequest(absl::Milliseconds(1)));
}
} // namespace testing
} // namespace grpc_core
int main(int argc, char** argv) {
grpc::testing::TestEnvironment env(&argc, argv);
::testing::InitGoogleTest(&argc, argv);

@ -575,14 +575,23 @@ grpc_cc_test(
grpc_cc_test(
name = "no_server_test",
srcs = ["no_server_test.cc"],
external_deps = [
"absl/status",
"absl/status:statusor",
"absl/time",
],
language = "C++",
deps = [
"cq_verifier",
"//:endpoint_addresses",
"//:exec_ctx",
"//:gpr",
"//:grpc_public_hdrs",
"//:grpc_resolver",
"//:grpc_resolver_fake",
"//:ref_counted_ptr",
"//src/core:channel_args",
"//src/core:grpc_service_config",
"//test/core/util:grpc_test_util",
],
)

@ -18,6 +18,12 @@
#include <string.h>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/time/time.h"
#include <grpc/grpc.h>
#include <grpc/grpc_security.h>
#include <grpc/impl/propagation_bits.h>
@ -27,8 +33,12 @@
#include <grpc/support/time.h>
#include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gprpp/ref_counted_ptr.h"
#include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/resolver/endpoint_addresses.h"
#include "src/core/lib/resolver/resolver.h"
#include "src/core/lib/service_config/service_config.h"
#include "test/core/end2end/cq_verifier.h"
#include "test/core/util/test_config.h"
@ -43,13 +53,12 @@ void run_test(bool wait_for_ready) {
grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
response_generator =
grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
grpc_arg arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
response_generator.get());
grpc_channel_args args = {1, &arg};
auto args = grpc_core::ChannelArgs().SetObject(response_generator).ToC();
// create a call, channel to a non existant server
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
grpc_channel* chan = grpc_channel_create("fake:nonexistant", creds, &args);
grpc_channel* chan =
grpc_channel_create("fake:nonexistant", creds, args.get());
grpc_channel_credentials_release(creds);
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(2);
grpc_call* call = grpc_channel_create_call(
@ -80,9 +89,13 @@ void run_test(bool wait_for_ready) {
grpc_core::CqVerifier::tag(1), nullptr));
{
response_generator->WaitForResolverSet();
response_generator->WaitForResolverSet(
absl::Seconds(5 * grpc_test_slowdown_factor()));
grpc_core::ExecCtx exec_ctx;
response_generator->SetFailure();
grpc_core::Resolver::Result result;
result.addresses = absl::UnavailableError("Resolver transient failure");
result.service_config = result.addresses.status();
response_generator->SetResponseSynchronously(std::move(result));
}
// verify that all tags get completed

@ -467,22 +467,17 @@ TEST_F(KeepaliveThrottlingTest, NewSubchannelsUseUpdatedKeepaliveTime) {
// response generator to switch between the two.
auto response_generator =
grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
grpc_arg client_args[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), 1 * 1000),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_BDP_PROBE), 0),
grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
response_generator.get())};
grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
client_args};
auto client_channel_args =
grpc_core::ChannelArgs()
.Set(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0)
.Set(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 0)
.Set(GRPC_ARG_KEEPALIVE_TIME_MS, 1 * 1000)
.Set(GRPC_ARG_HTTP2_BDP_PROBE, 0)
.SetObject(response_generator)
.ToC();
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
grpc_channel* channel =
grpc_channel_create("fake:///", creds, &client_channel_args);
grpc_channel_create("fake:///", creds, client_channel_args.get());
grpc_channel_credentials_release(creds);
// For a single subchannel 3 GOAWAYs would be sufficient to increase the
// keepalive time from 1 second to beyond 5 seconds. Even though we are
@ -539,22 +534,17 @@ TEST_F(KeepaliveThrottlingTest,
// create a single channel with round robin load balancing policy.
auto response_generator =
grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
grpc_arg client_args[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 0),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), 1 * 1000),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_HTTP2_BDP_PROBE), 0),
grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
response_generator.get())};
grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
client_args};
auto client_channel_args =
grpc_core::ChannelArgs()
.Set(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0)
.Set(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 0)
.Set(GRPC_ARG_KEEPALIVE_TIME_MS, 1 * 1000)
.Set(GRPC_ARG_HTTP2_BDP_PROBE, 0)
.SetObject(response_generator)
.ToC();
grpc_channel_credentials* creds = grpc_insecure_credentials_create();
grpc_channel* channel =
grpc_channel_create("fake:///", creds, &client_channel_args);
grpc_channel_create("fake:///", creds, client_channel_args.get());
grpc_channel_credentials_release(creds);
response_generator->SetResponseSynchronously(
BuildResolverResult({absl::StrCat("ipv4:", server_address1),

@ -213,29 +213,16 @@ class FakeResolverResponseGeneratorWrapper {
response_generator_ = std::move(other.response_generator_);
}
void SetResponse(grpc_core::Resolver::Result result) {
grpc_core::ExecCtx exec_ctx;
response_generator_->SetResponseSynchronously(std::move(result));
}
void SetNextResolution(const std::vector<int>& ports,
const char* service_config_json = nullptr,
const grpc_core::ChannelArgs& per_address_args =
grpc_core::ChannelArgs()) {
grpc_core::ExecCtx exec_ctx;
response_generator_->SetResponseSynchronously(
BuildFakeResults(ports, service_config_json, per_address_args));
}
void SetNextResolutionUponError(const std::vector<int>& ports) {
grpc_core::ExecCtx exec_ctx;
response_generator_->SetReresolutionResponseSynchronously(
BuildFakeResults(ports));
}
void SetFailureOnReresolution() {
grpc_core::ExecCtx exec_ctx;
response_generator_->SetFailureOnReresolution();
}
void SetResponse(grpc_core::Resolver::Result result) {
grpc_core::ExecCtx exec_ctx;
response_generator_->SetResponseSynchronously(std::move(result));
SetResponse(BuildFakeResults(ports, service_config_json, per_address_args));
}
grpc_core::FakeResolverResponseGenerator* Get() const {
@ -1155,10 +1142,15 @@ TEST_F(PickFirstTest, ReresolutionNoSelected) {
DEBUG_LOCATION, stub, StatusCode::UNAVAILABLE,
MakeConnectionFailureRegex("failed to connect to all addresses"));
}
// Set a re-resolution result that contains reachable ports, so that the
// PF should request re-resolution.
gpr_log(GPR_INFO, "****** WAITING FOR RE-RESOLUTION *******");
EXPECT_TRUE(response_generator.Get()->WaitForReresolutionRequest(
absl::Seconds(5 * grpc_test_slowdown_factor())));
gpr_log(GPR_INFO, "****** RE-RESOLUTION SEEN *******");
// Send a resolver result that contains reachable ports, so that the
// pick_first LB policy can recover soon.
response_generator.SetNextResolutionUponError(alive_ports);
gpr_log(GPR_INFO, "****** RE-RESOLUTION SET *******");
response_generator.SetNextResolution(alive_ports);
gpr_log(GPR_INFO, "****** RE-RESOLUTION SENT *******");
WaitForServer(DEBUG_LOCATION, stub, 0, [](const Status& status) {
EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
EXPECT_THAT(status.error_message(),
@ -1294,7 +1286,6 @@ TEST_F(PickFirstTest, IdleOnDisconnect) {
CheckRpcSendOk(DEBUG_LOCATION, stub);
EXPECT_EQ(channel->GetState(false), GRPC_CHANNEL_READY);
// Stop server. Channel should go into state IDLE.
response_generator.SetFailureOnReresolution();
servers_[0]->Shutdown();
EXPECT_TRUE(WaitForChannelNotReady(channel.get()));
EXPECT_EQ(channel->GetState(false), GRPC_CHANNEL_IDLE);
@ -1603,16 +1594,20 @@ TEST_F(RoundRobinTest, ReresolveOnSubchannelConnectionFailure) {
response_generator.SetNextResolution(ports);
// Wait for both servers to be seen.
WaitForServers(DEBUG_LOCATION, stub, 0, 2);
// Tell the fake resolver to send an update that adds the last server, but
// only when the LB policy requests re-resolution.
ports.push_back(servers_[2]->port_);
response_generator.SetNextResolutionUponError(ports);
// Have server 0 send a GOAWAY. This should trigger a re-resolution.
gpr_log(GPR_INFO, "****** SENDING GOAWAY FROM SERVER 0 *******");
{
grpc_core::ExecCtx exec_ctx;
grpc_core::Server::FromC(servers_[0]->server_->c_server())->SendGoaways();
}
gpr_log(GPR_INFO, "****** WAITING FOR RE-RESOLUTION REQUEST *******");
EXPECT_TRUE(response_generator.Get()->WaitForReresolutionRequest(
absl::Seconds(5 * grpc_test_slowdown_factor())));
gpr_log(GPR_INFO, "****** RE-RESOLUTION REQUEST SEEN *******");
// Tell the fake resolver to send an update that adds the last server, but
// only when the LB policy requests re-resolution.
ports.push_back(servers_[2]->port_);
response_generator.SetNextResolution(ports);
// Wait for the client to see server 2.
WaitForServer(DEBUG_LOCATION, stub, 2);
}

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save