Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

xds end2end tests: change tests to check all RPC failure statuses (#29974)

Browse Source 
* WIP

* fix from merge

* fix ring_hash tests

* ring hash: fix picker propagation bug in xds_cluster_manager policy

* fix build

* clang-format (ish)

* fix build

* more conversion

* remove old CheckRpcSendFailure

* fix SendRpcsAndCountFailuresWithMessage

* fix WaitForBackend

* clang-format

* revert xds resolver change, that will be done in a separate PR

* Automated change: Fix sanity tests

Co-authored-by: markdroth <markdroth@users.noreply.github.com>
pull/29980/head
Mark D. Roth 3 years ago committed by GitHub
parent 475623c489
commit 8459824e5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 672 additions and 699 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 180
      test/cpp/end2end/xds/xds_cluster_end2end_test.cc
  2. 46
      test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc
  3. 131
      test/cpp/end2end/xds/xds_core_end2end_test.cc
  4. 48
      test/cpp/end2end/xds/xds_csds_end2end_test.cc
  5. 40
      test/cpp/end2end/xds/xds_end2end_test.cc
  6. 122
      test/cpp/end2end/xds/xds_end2end_test_lib.cc
  7. 67
      test/cpp/end2end/xds/xds_end2end_test_lib.h
  8. 14
      test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
  9. 366
      test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
  10. 47
      test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
  11. 8
      test/cpp/end2end/xds/xds_rls_end2end_test.cc
  12. 302
      test/cpp/end2end/xds/xds_routing_end2end_test.cc

180
test/cpp/end2end/xds/xds_cluster_end2end_test.cc
Unescape View File

@ -39,6 +39,7 @@ using ClientStats = LrsServiceImpl::ClientStats;
constexpr char kLbDropType[] = "lb";
constexpr char kThrottleDropType[] = "throttle";
constexpr char kStatusMessageDropPrefix[] = "EDS-configured drop: ";
//
// CDS tests
@ -239,11 +240,14 @@ TEST_P(CdsTest, ClusterRemoved) {
// Unset CDS resource.
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
// Wait for RPCs to start failing.
do {
} while (SendRpc(RpcOptions(), nullptr).ok());
// Make sure RPCs are still failing.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(1000));
SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
EXPECT_EQ(absl::StrCat("CDS resource \"", kDefaultClusterName,
"\" does not exist"),
result.status.error_message());
return false;
});
// Make sure we ACK'ed the update.
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
@ -349,23 +353,7 @@ TEST_P(CdsTest, ClusterChangeAfterAdsCallFails) {
cluster.mutable_eds_cluster_config()->set_service_name(kNewEdsResourceName);
balancer_->ads_service()->SetCdsResource(cluster);
// Make sure client sees the change.
// TODO(roth): This should not be allowing errors. The errors are
// being caused by a bug that triggers in the following situation:
//
// 1. xDS call fails.
// 2. When xDS call is restarted, the server sends the updated CDS
// resource that points to the new EDS resource name.
// 3. When the client receives the CDS update, it does two things:
// - Sends the update to the CDS LB policy, which creates a new
// xds_cluster_resolver policy using the new EDS service name.
// - Notices that the CDS update no longer refers to the old EDS
// service name, so removes that resource, notifying the old
// xds_cluster_resolver policy that the resource no longer exists.
//
// Need to figure out a way to fix this bug, and then change this to
// not allow failures.
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 1);
}
//
@ -455,13 +443,21 @@ TEST_P(EdsTest, InitiallyEmptyServerlist) {
EdsResourceArgs args({std::move(empty_locality)});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// RPCs should fail.
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Send non-empty serverlist.
args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends()}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// RPCs should eventually succeed.
WaitForAllBackends(DEBUG_LOCATION, 0, 1,
WaitForBackendOptions().set_allow_failures(true));
WaitForAllBackends(DEBUG_LOCATION, 0, 1, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
// Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if
@ -493,17 +489,16 @@ TEST_P(EdsTest, BackendsRestart) {
WaitForAllBackends(DEBUG_LOCATION);
// Stop backends. RPCs should fail.
ShutdownAllBackends();
// Sending multiple failed requests instead of just one to ensure that the
// client notices that all backends are down before we restart them. If we
// didn't do this, then a single RPC could fail here due to the race
// condition between the LB pick and the GOAWAY from the chosen backend
// being shut down, which would not actually prove that the client noticed
// that all of the backends are down. Then, when we send another request
// below (which we expect to succeed), if the callbacks happen in the wrong
// order, the same race condition could happen again due to the client not
// yet having noticed that the backends were all down.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(backends_.size()));
// Wait for channel to report TRANSIENT_FAILURE.
EXPECT_TRUE(channel_->WaitForStateChange(
GRPC_CHANNEL_READY, grpc_timeout_seconds_to_deadline(5)));
EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false));
// RPCs should fail.
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
// Restart all backends. RPCs should start succeeding again.
StartAllBackends();
CheckRpcSendOk(DEBUG_LOCATION, 1,
@ -672,7 +667,7 @@ TEST_P(EdsTest, ManyLocalitiesStressTest) {
}
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Wait until backend 0 is ready.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
@ -837,7 +832,8 @@ TEST_P(EdsTest, Drops) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
@ -858,7 +854,8 @@ TEST_P(EdsTest, DropPerHundred) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate,
@ -879,7 +876,8 @@ TEST_P(EdsTest, DropPerTenThousand) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate,
@ -907,7 +905,8 @@ TEST_P(EdsTest, DropConfigUpdate) {
// Send kNumRpcsLbOnly RPCs and count the drops.
gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcsLbOnly, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcsLbOnly, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
// The drop rate should be roughly equal to the expectation.
double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsLbOnly;
@ -924,24 +923,27 @@ TEST_P(EdsTest, DropConfigUpdate) {
const double kDropRateThreshold =
(kDropRateForLb + kDropRateForLbAndThrottle) / 2;
size_t num_rpcs = kNumRpcsBoth;
while (seen_drop_rate < kDropRateThreshold) {
EchoResponse response;
const Status status = SendRpc(RpcOptions(), &response);
++num_rpcs;
if (!status.ok() &&
absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
++num_drops;
} else {
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
EXPECT_EQ(response.message(), kRequestMessage);
}
seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
}
SendRpcsUntil(
DEBUG_LOCATION,
[&](const RpcResult& result) {
++num_rpcs;
if (result.status.ok()) {
EXPECT_EQ(result.response.message(), kRequestMessage);
} else {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_THAT(result.status.error_message(),
::testing::StartsWith(kStatusMessageDropPrefix));
++num_drops;
}
seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
return seen_drop_rate < kDropRateThreshold;
},
/*timeout_ms=*/20000);
// Send kNumRpcsBoth RPCs and count the drops.
gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
num_drops = SendRpcsAndCountFailuresWithMessage(DEBUG_LOCATION, kNumRpcsBoth,
"EDS-configured drop: ");
StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
// The new drop rate should be roughly equal to the expectation.
seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsBoth;
@ -962,7 +964,8 @@ TEST_P(EdsTest, DropAll) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and all of them are dropped.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: ");
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
EXPECT_EQ(num_drops, kNumRpcs);
}
@ -997,7 +1000,7 @@ TEST_P(FailoverTest, ChooseHighestPriority) {
0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
for (size_t i = 0; i < 3; ++i) {
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
@ -1017,7 +1020,7 @@ TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) {
{"locality3", {}, kDefaultLocalityWeight, 0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
for (size_t i = 1; i < 3; ++i) {
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
@ -1049,7 +1052,7 @@ TEST_P(FailoverTest, Failover) {
{"locality3", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
}
@ -1071,16 +1074,10 @@ TEST_P(FailoverTest, SwitchBackToHigherPriority) {
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3);
ShutdownBackend(3);
backends_[3]->StopListeningAndSendGoaways();
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
ShutdownBackend(0);
WaitForBackend(
DEBUG_LOCATION, 1,
WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
true));
for (size_t i = 0; i < backends_.size(); ++i) {
if (i == 1) continue;
EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
}
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
@ -1096,7 +1093,11 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) {
{"locality1", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 1},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
args = EdsResourceArgs({
{"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
0},
@ -1104,8 +1105,12 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) {
1},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
// Tests that after the localities' priorities are updated, we still choose
@ -1124,7 +1129,7 @@ TEST_P(FailoverTest, UpdatePriority) {
0},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0U, backends_[0]->backend_service()->request_count());
EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
@ -1161,7 +1166,7 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) {
// When we get the first update, all backends in priority 0 are down,
// so we will create priority 1. Backends 0 and 1 should have traffic,
// but backend 2 should not.
WaitForAllBackends(DEBUG_LOCATION, 0, 2,
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count());
// Second update:
@ -1227,7 +1232,7 @@ TEST_P(FailoverTest, PriorityChildNameChurn) {
2},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
WaitForBackend(DEBUG_LOCATION, 3,
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
// P2 should not have gotten any traffic in this change.
EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count());
@ -1255,14 +1260,15 @@ TEST_P(ClientLoadReportingTest, Vanilla) {
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Wait until all backends are ready.
size_t num_warmup_rpcs = WaitForAllBackends(
DEBUG_LOCATION, 0, 4, WaitForBackendOptions().set_reset_counters(false));
size_t num_warmup_rpcs =
WaitForAllBackends(DEBUG_LOCATION, 0, 4, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
// Send kNumRpcsPerAddress RPCs per server.
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size());
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(kNumFailuresPerAddress * backends_.size())
.set_rpc_options(RpcOptions().set_server_fail(true)));
for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "",
RpcOptions().set_server_fail(true));
}
const size_t total_successful_rpcs_sent =
(kNumRpcsPerAddress * backends_.size()) + num_warmup_rpcs;
const size_t total_failed_rpcs_sent =
@ -1321,10 +1327,10 @@ TEST_P(ClientLoadReportingTest, SendAllClusters) {
size_t num_warmup_rpcs = WaitForAllBackends(DEBUG_LOCATION);
// Send kNumRpcsPerAddress RPCs per server.
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size());
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(kNumFailuresPerAddress * backends_.size())
.set_rpc_options(RpcOptions().set_server_fail(true)));
for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "",
RpcOptions().set_server_fail(true));
}
// Check that each backend got the right number of requests.
for (size_t i = 0; i < backends_.size(); ++i) {
EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
@ -1552,7 +1558,6 @@ TEST_P(ClientLoadReportingTest, DropStats) {
kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
const size_t kNumRpcs =
ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
const char kStatusMessageDropPrefix[] = "EDS-configured drop: ";
// The ADS response contains two drop categories.
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
@ -1560,7 +1565,8 @@ TEST_P(ClientLoadReportingTest, DropStats) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Send kNumRpcs RPCs and count the drops.
size_t num_drops = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, kStatusMessageDropPrefix);
DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE,
kStatusMessageDropPrefix);
// The drop rate should be roughly equal to the expectation.
const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,

46
test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc
Unescape View File

@ -286,7 +286,7 @@ using AggregateClusterTest = ClusterTypeTest;
INSTANTIATE_TEST_SUITE_P(XdsTest, AggregateClusterTest,
::testing::Values(XdsTestType()), &XdsTestType::Name);
TEST_P(AggregateClusterTest, ) {
TEST_P(AggregateClusterTest, Basic) {
CreateAndStartBackends(2);
const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1";
@ -326,13 +326,13 @@ TEST_P(AggregateClusterTest, ) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
// Bring backend 0 back and ensure all traffic goes back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -386,13 +386,13 @@ TEST_P(AggregateClusterTest, DiamondDependency) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -522,8 +522,8 @@ TEST_P(AggregateClusterTest, FallBackWithConnectivityChurn) {
connection_attempt_injector.Start();
// Wait for P0 backend.
// Increase timeout to account for subchannel connection delays.
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(),
RpcOptions().set_timeout_ms(2000));
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), RpcOptions().set_timeout_ms(2000));
// Send GOAWAY from the P0 backend.
// We don't actually shut it down here to avoid flakiness caused by
// failing an RPC after the client has already sent it but before the
@ -583,13 +583,13 @@ TEST_P(AggregateClusterTest, EdsToLogicalDns) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -644,13 +644,13 @@ TEST_P(AggregateClusterTest, LogicalDnsToEds) {
// Wait for traffic to go to backend 0.
WaitForBackend(DEBUG_LOCATION, 0);
// Shutdown backend 0 and wait for all traffic to go to backend 1.
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true));
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1);
auto response_state = balancer_->ads_service()->cds_response_state();
ASSERT_TRUE(response_state.has_value());
EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
// Bring backend 0 back and ensure all traffic go back to it.
ShutdownBackend(0);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0);
}
@ -713,7 +713,11 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) {
std::move(result));
}
// When an RPC fails, we know the channel has seen the update.
CheckRpcSendFailure(DEBUG_LOCATION);
constexpr char kErrorMessage[] =
// TODO(roth): Figure out how to get some sort of resolution note
// included here as part of https://github.com/grpc/grpc/issues/22883.
"empty address list: ";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Send an EDS update that moves locality1 to priority 0.
args1 = EdsResourceArgs({
{"locality1", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
@ -723,8 +727,12 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) {
});
balancer_->ads_service()->SetEdsResource(
BuildEdsResource(args1, kNewEdsService1Name));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
});
}
TEST_P(AggregateClusterTest, MultipleClustersWithSameLocalities) {

131
test/cpp/end2end/xds/xds_core_end2end_test.cc
Unescape View File

@ -198,9 +198,10 @@ TEST_P(XdsClientTest, MultipleBadCdsResources) {
{"cluster", kClusterName2},
};
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_metadata(std::move(metadata_cluster_2))));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
"cluster_name_2: UNAVAILABLE: invalid resource: INVALID_ARGUMENT:.*"
"errors parsing CDS resource.*DiscoveryType is not valid.*",
RpcOptions().set_metadata(std::move(metadata_cluster_2)));
}
TEST_P(XdsClientTest, XdsStreamErrorPropagation) {
@ -394,16 +395,20 @@ INSTANTIATE_TEST_SUITE_P(
TEST_P(TimeoutTest, LdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kLdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, LdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
@ -417,8 +422,8 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000));
// Create second channel for a new server name.
// This should fail because there is no LDS resource for this server name.
auto channel2 =
CreateChannel(/*failover_timeout_ms=*/0, "new-server.example.com");
const char* kNewServerName = "new-server.example.com";
auto channel2 = CreateChannel(/*failover_timeout_ms=*/0, kNewServerName);
auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
ClientContext context;
EchoRequest request;
@ -428,21 +433,29 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) {
auto status =
SendRpcMethod(stub2.get(), rpc_options, &context, request, &response);
EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
EXPECT_THAT(status.error_message(),
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
}
TEST_P(TimeoutTest, RdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kRdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, RdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kRdsTypeUrl,
kDefaultRouteConfigurationName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
@ -456,12 +469,13 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000));
// Add listener for 2nd channel, but no RDS resource.
const char* kNewServerName = "new-server.example.com";
const char* kNewRouteConfigName = "rds_resource_does_not_exist";
Listener listener = default_listener_;
listener.set_name(kNewServerName);
HttpConnectionManager http_connection_manager =
ClientHcmAccessor().Unpack(listener);
auto* rds = http_connection_manager.mutable_rds();
rds->set_route_config_name("rds_resource_does_not_exist");
rds->set_route_config_name(kNewRouteConfigName);
rds->mutable_config_source()->mutable_self();
ClientHcmAccessor().Pack(http_connection_manager, &listener);
balancer_->ads_service()->SetLdsResource(listener);
@ -478,20 +492,26 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) {
auto status =
SendRpcMethod(stub2.get(), rpc_options, &context, request, &response);
EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
EXPECT_EQ(status.error_message(),
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
}
TEST_P(TimeoutTest, CdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kCdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, CdsResourceNotPresentInRequest) {
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) {
@ -509,31 +529,38 @@ TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) {
balancer_->ads_service()->SetRdsResource(route_config);
// New cluster times out.
// May need to wait a bit for the change to propagate to the client.
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
bool error_seen = false;
do {
auto status = SendRpc();
if (status.error_code() == StatusCode::UNAVAILABLE) {
error_seen = true;
break;
}
} while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0);
EXPECT_TRUE(error_seen);
SendRpcsUntil(
DEBUG_LOCATION,
[&](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
EXPECT_EQ(absl::StrCat("CDS resource \"", kNewClusterName,
"\" does not exist"),
result.status.error_message());
return false;
},
/*timeout_ms=*/30000, RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsServerIgnoresRequest) {
balancer_->ads_service()->IgnoreResourceType(kEdsTypeUrl);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsResourceNotPresentInRequest) {
// No need to remove EDS resource, since the test suite does not add it
// by default.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_rpc_options(
RpcOptions().set_timeout_ms(4000)));
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(4000));
}
TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) {
@ -557,16 +584,20 @@ TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) {
balancer_->ads_service()->SetRdsResource(route_config);
// New EDS resource times out.
// May need to wait a bit for the RDS change to propagate to the client.
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30);
bool error_seen = false;
do {
auto status = SendRpc(RpcOptions().set_rpc_method(METHOD_ECHO1));
if (status.error_code() == StatusCode::UNAVAILABLE) {
error_seen = true;
break;
}
} while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0);
EXPECT_TRUE(error_seen);
SendRpcsUntil(
DEBUG_LOCATION,
[](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code());
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
EXPECT_EQ(
result.status.error_message(),
"weighted_target: all children report state TRANSIENT_FAILURE");
return false;
},
/*timeout_ms=*/30000,
RpcOptions().set_rpc_method(METHOD_ECHO1).set_timeout_ms(4000));
}
TEST_P(TimeoutTest, ServerDoesNotResendAfterAdsStreamRestart) {

48
test/cpp/end2end/xds/xds_csds_end2end_test.cc
Unescape View File

@ -560,11 +560,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) {
TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) {
int kTimeoutMillisecond = 1000;
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -591,11 +589,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) {
routes2->mutable_route()->set_cluster(kClusterName2);
SetRouteConfiguration(balancer_.get(), route_config);
// Try to get the configs plumb through
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::AllOf(
@ -639,10 +635,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -656,10 +652,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) {
balancer_->ads_service()->UnsetResource(kRdsTypeUrl,
kDefaultRouteConfigurationName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(
csds_response.config(0).generic_xds_configs(),
@ -672,10 +668,9 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"),
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(csds_response.config(0).generic_xds_configs(),
::testing::Contains(EqGenericXdsConfig(
@ -687,10 +682,11 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) {
int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure.
balancer_->ads_service()->UnsetResource(kEdsTypeUrl, kDefaultEdsServiceName);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
.set_expected_error_code(grpc::StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE",
RpcOptions().set_timeout_ms(kTimeoutMillisecond));
auto csds_response = FetchCsdsResponse();
EXPECT_THAT(
csds_response.config(0).generic_xds_configs(),

40
test/cpp/end2end/xds/xds_end2end_test.cc
Unescape View File

@ -369,8 +369,16 @@ class XdsSecurityTest : public XdsEnd2endTest {
continue;
}
} else {
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state "
"TRANSIENT_FAILURE");
}
});
Status status = SendRpc();
if (!status.ok()) {
gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
@ -768,11 +776,15 @@ TEST_P(XdsSecurityTest, TestTlsConfigurationInCombinedValidationContext) {
->set_instance_name("fake_plugin1");
transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
balancer_->ads_service()->SetCdsResource(cluster);
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
Status status = SendRpc();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
}
});
}
// TODO(yashykt): Remove this test once we stop supporting old fields
@ -789,11 +801,15 @@ TEST_P(XdsSecurityTest,
->set_instance_name("fake_plugin1");
transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
balancer_->ads_service()->SetCdsResource(cluster);
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackendOptions().set_allow_failures(true));
Status status = SendRpc();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE");
}
});
}
TEST_P(XdsSecurityTest, TestMtlsConfigurationWithNoSanMatchers) {

122
test/cpp/end2end/xds/xds_end2end_test_lib.cc
Unescape View File

@ -834,20 +834,41 @@ Status XdsEnd2endTest::SendRpc(const RpcOptions& rpc_options,
return status;
}
void XdsEnd2endTest::CheckRpcSendOk(
const grpc_core::DebugLocation& debug_location, const size_t times,
void XdsEnd2endTest::SendRpcsUntil(
const grpc_core::DebugLocation& debug_location,
std::function<bool(const RpcResult&)> continue_predicate, int timeout_ms,
const RpcOptions& rpc_options) {
for (size_t i = 0; i < times; ++i) {
EchoResponse response;
const Status status = SendRpc(rpc_options, &response);
EXPECT_TRUE(status.ok())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
absl::Time deadline = absl::InfiniteFuture();
if (timeout_ms != 0) {
deadline = absl::Now() +
(absl::Milliseconds(timeout_ms) * grpc_test_slowdown_factor());
}
while (true) {
RpcResult result;
result.status = SendRpc(rpc_options, &result.response);
if (!continue_predicate(result)) return;
EXPECT_LE(absl::Now(), deadline)
<< debug_location.file() << ":" << debug_location.line();
EXPECT_EQ(response.message(), kRequestMessage);
if (absl::Now() >= deadline) break;
}
}
void XdsEnd2endTest::CheckRpcSendOk(
const grpc_core::DebugLocation& debug_location, const size_t times,
const RpcOptions& rpc_options) {
SendRpcsUntil(
debug_location,
[debug_location, times, n = size_t(0)](const RpcResult& result) mutable {
EXPECT_TRUE(result.status.ok())
<< "code=" << result.status.error_code()
<< " message=" << result.status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
EXPECT_EQ(result.response.message(), kRequestMessage);
return ++n < times;
},
/*timeout_ms=*/0, rpc_options);
}
void XdsEnd2endTest::CheckRpcSendFailure(
const grpc_core::DebugLocation& debug_location, StatusCode expected_status,
absl::string_view expected_message_regex, const RpcOptions& rpc_options) {
@ -857,41 +878,29 @@ void XdsEnd2endTest::CheckRpcSendFailure(
EXPECT_EQ(expected_status, status.error_code())
<< debug_location.file() << ":" << debug_location.line();
EXPECT_THAT(status.error_message(),
::testing::ContainsRegex(expected_message_regex))
::testing::MatchesRegex(expected_message_regex))
<< debug_location.file() << ":" << debug_location.line();
}
void XdsEnd2endTest::CheckRpcSendFailure(
const grpc_core::DebugLocation& debug_location,
const CheckRpcSendFailureOptions& options) {
for (size_t i = 0; options.continue_predicate(i); ++i) {
const Status status = SendRpc(options.rpc_options);
EXPECT_FALSE(status.ok())
<< " at " << debug_location.file() << ":" << debug_location.line();
if (options.expected_error_code != StatusCode::OK) {
EXPECT_EQ(options.expected_error_code, status.error_code())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
}
}
}
size_t XdsEnd2endTest::SendRpcsAndCountFailuresWithMessage(
const grpc_core::DebugLocation& debug_location, size_t num_rpcs,
const char* drop_error_message_prefix, const RpcOptions& rpc_options) {
StatusCode expected_status, absl::string_view expected_message_prefix,
const RpcOptions& rpc_options) {
size_t num_failed = 0;
for (size_t i = 0; i < num_rpcs; ++i) {
Status status = SendRpc(rpc_options);
if (!status.ok()) {
EXPECT_THAT(status.error_message(),
::testing::StartsWith(drop_error_message_prefix))
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
++num_failed;
}
}
SendRpcsUntil(
debug_location,
[&, n = size_t(0)](const RpcResult& result) mutable {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), expected_status)
<< debug_location.file() << ":" << debug_location.line();
EXPECT_THAT(result.status.error_message(),
::testing::StartsWith(expected_message_prefix))
<< debug_location.file() << ":" << debug_location.line();
++num_failed;
}
return ++n < num_rpcs;
},
/*timeout_ms=*/0, rpc_options);
return num_failed;
}
@ -954,28 +963,29 @@ std::vector<XdsEnd2endTest::ConcurrentRpc> XdsEnd2endTest::SendConcurrentRpcs(
size_t XdsEnd2endTest::WaitForAllBackends(
const grpc_core::DebugLocation& debug_location, size_t start_index,
size_t stop_index, const WaitForBackendOptions& wait_options,
const RpcOptions& rpc_options) {
size_t num_rpcs = 0;
auto deadline = absl::Now() + (absl::Milliseconds(wait_options.timeout_ms) *
grpc_test_slowdown_factor());
size_t stop_index, std::function<void(const RpcResult&)> check_status,
const WaitForBackendOptions& wait_options, const RpcOptions& rpc_options) {
if (check_status == nullptr) {
check_status = [&](const RpcResult& result) {
EXPECT_TRUE(result.status.ok())
<< "code=" << result.status.error_code()
<< " message=" << result.status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
};
}
gpr_log(GPR_INFO,
"========= WAITING FOR BACKENDS [%" PRIuPTR ", %" PRIuPTR
") ==========",
start_index, stop_index);
while (!SeenAllBackends(start_index, stop_index, rpc_options.service)) {
Status status = SendRpc(rpc_options);
if (!wait_options.allow_failures) {
EXPECT_TRUE(status.ok())
<< "code=" << status.error_code()
<< " message=" << status.error_message() << " at "
<< debug_location.file() << ":" << debug_location.line();
}
EXPECT_LE(absl::Now(), deadline)
<< " at " << debug_location.file() << ":" << debug_location.line();
if (absl::Now() >= deadline) break;
++num_rpcs;
}
size_t num_rpcs = 0;
SendRpcsUntil(
debug_location,
[&](const RpcResult& result) {
++num_rpcs;
check_status(result);
return !SeenAllBackends(start_index, stop_index, rpc_options.service);
},
wait_options.timeout_ms, rpc_options);
if (wait_options.reset_counters) ResetBackendCounters();
gpr_log(GPR_INFO, "Backends up; sent %" PRIuPTR " warm up requests",
num_rpcs);

67
test/cpp/end2end/xds/xds_end2end_test_lib.h
Unescape View File

@ -801,6 +801,17 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
GPR_UNREACHABLE_CODE(return grpc::Status::OK);
}
// Send RPCs in a loop until either continue_predicate() returns false
// or timeout_ms elapses.
struct RpcResult {
Status status;
EchoResponse response;
};
void SendRpcsUntil(const grpc_core::DebugLocation& debug_location,
std::function<bool(const RpcResult&)> continue_predicate,
int timeout_ms = 5000,
const RpcOptions& rpc_options = RpcOptions());
// Sends the specified number of RPCs and fails if the RPC fails.
void CheckRpcSendOk(const grpc_core::DebugLocation& debug_location,
const size_t times = 1,
@ -813,49 +824,12 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
absl::string_view expected_message_regex,
const RpcOptions& rpc_options = RpcOptions());
// DEPRECATED -- USE THE ABOVE VARIANT INSTEAD.
// TODO(roth): Change all existing callers to use the above variant
// instead and then remove this.
struct CheckRpcSendFailureOptions {
std::function<bool(size_t)> continue_predicate = [](size_t i) {
return i < 1;
};
RpcOptions rpc_options;
StatusCode expected_error_code = StatusCode::OK;
CheckRpcSendFailureOptions() {}
CheckRpcSendFailureOptions& set_times(size_t times) {
continue_predicate = [times](size_t i) { return i < times; };
return *this;
}
CheckRpcSendFailureOptions& set_continue_predicate(
std::function<bool(size_t)> pred) {
continue_predicate = std::move(pred);
return *this;
}
CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) {
rpc_options = options;
return *this;
}
CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) {
expected_error_code = code;
return *this;
}
};
void CheckRpcSendFailure(
const grpc_core::DebugLocation& debug_location,
const CheckRpcSendFailureOptions& options = CheckRpcSendFailureOptions());
// Sends num_rpcs RPCs, counting how many of them fail with a message
// matching the specfied drop_error_message_prefix.
// Any failure with a non-matching message is a test failure.
// matching the specfied expected_message_prefix.
// Any failure with a non-matching status or message is a test failure.
size_t SendRpcsAndCountFailuresWithMessage(
const grpc_core::DebugLocation& debug_location, size_t num_rpcs,
const char* drop_error_message_prefix,
StatusCode expected_status, absl::string_view expected_message_prefix,
const RpcOptions& rpc_options = RpcOptions());
// A class for running a long-running RPC in its own thread.
@ -901,8 +875,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
struct WaitForBackendOptions {
// If true, resets the backend counters before returning.
bool reset_counters = true;
// If true, RPC failures will not cause the test to fail.
bool allow_failures = false;
// How long to wait for the backend(s) to see requests.
int timeout_ms = 5000;
@ -913,11 +885,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
return *this;
}
WaitForBackendOptions& set_allow_failures(bool enable) {
allow_failures = enable;
return *this;
}
WaitForBackendOptions& set_timeout_ms(int ms) {
timeout_ms = ms;
return *this;
@ -925,20 +892,24 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
};
// Sends RPCs until all of the backends in the specified range see requests.
// The check_status callback will be invoked to check the status of
// every RPC; if null, the default is to check that the RPC succeeded.
// Returns the total number of RPCs sent.
size_t WaitForAllBackends(
const grpc_core::DebugLocation& debug_location, size_t start_index = 0,
size_t stop_index = 0,
std::function<void(const RpcResult&)> check_status = nullptr,
const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
const RpcOptions& rpc_options = RpcOptions());
// Sends RPCs until the backend at index backend_idx sees requests.
void WaitForBackend(
const grpc_core::DebugLocation& debug_location, size_t backend_idx,
std::function<void(const RpcResult&)> check_status = nullptr,
const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
const RpcOptions& rpc_options = RpcOptions()) {
WaitForAllBackends(debug_location, backend_idx, backend_idx + 1,
wait_options, rpc_options);
check_status, wait_options, rpc_options);
}
//

14
test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
Unescape View File

@ -116,12 +116,10 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) {
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Fire several RPCs, and expect all of them to be aborted.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_times(5)
.set_rpc_options(RpcOptions().set_wait_for_ready(true))
.set_expected_error_code(StatusCode::ABORTED));
for (size_t i = 0; i < 5; ++i) {
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::ABORTED, "Fault injected",
RpcOptions().set_wait_for_ready(true));
}
}
// Without the listener config, the fault injection won't be enabled.
@ -166,7 +164,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) {
SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the aborts.
size_t num_aborted = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "Fault injected");
DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected");
// The abort rate should be roughly equal to the expectation.
const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
EXPECT_THAT(seen_abort_rate,
@ -196,7 +194,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
{"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)},
};
size_t num_aborted = SendRpcsAndCountFailuresWithMessage(
DEBUG_LOCATION, kNumRpcs, "Fault injected",
DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected",
RpcOptions().set_metadata(metadata));
// The abort rate should be roughly equal to the expectation.
const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;

366
test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
Unescape View File

@ -99,17 +99,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEjectionAndUnejection) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend is ejected, rpc destinated to it are now hashed to the other
@ -177,28 +175,24 @@ TEST_P(OutlierDetectionTest, SuccessRateMaxPercent) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options2);
CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options3);
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
@ -280,17 +274,15 @@ TEST_P(OutlierDetectionTest, SuccessRateStdevFactor) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the stdev_factor is high, no
@ -351,17 +343,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEnforcementPercentage) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the enforcement percentage is 0, no
@ -421,17 +411,15 @@ TEST_P(OutlierDetectionTest, SuccessRateMinimumHosts) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -492,17 +480,15 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -567,18 +553,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEjectionAndUnejection) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend is ejected all traffic going to the ejected backend should now
@ -646,28 +630,24 @@ TEST_P(OutlierDetectionTest, FailurePercentageMaxPercentage) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata1))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend should be ejected, trafficed picked up by another backend.
@ -743,18 +723,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageThreshold) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced 1 failure, but since the threshold is 50 % no
@ -815,18 +793,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEnforcementPercentage) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since the enforcement percentage is 0, no
@ -886,18 +862,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageMinimumHosts) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -959,18 +933,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageRequestVolume) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// All traffic still reaching the original backends and no backends are
@ -1050,10 +1022,14 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1
// outlier detection interval to pass. The 2 errors to the 1 backend will make
// exactly 1 outlier from the success rate algorithm; all 4 errors will make 3
@ -1063,33 +1039,21 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
// eject another backend because of failure percentage we will stop as we have
// reached our 50% limit.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
@ -1149,17 +1113,15 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentageBothDisabled) {
{"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
const auto rpc_options = RpcOptions().set_metadata(metadata);
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
// Cause an error and wait for 1 outlier detection interval to pass
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions()
.set_metadata(std::move(metadata))
.set_server_expected_error(StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
// 1 backend experenced failure, but since there is no counting there is no
@ -1235,42 +1197,34 @@ TEST_P(OutlierDetectionTest,
const auto rpc_options1 = RpcOptions().set_metadata(metadata1);
const auto rpc_options2 = RpcOptions().set_metadata(metadata2);
const auto rpc_options3 = RpcOptions().set_metadata(metadata3);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
// Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1
// outlier detection interval to pass. The errors should have caused 2
// ejctionss but since the policy is disabled we are not ejecting any and
// traffic flow as usual and RPCs reach destinated backends.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata1).set_server_expected_error(
StatusCode::CANCELLED));
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_metadata(metadata2).set_server_expected_error(
StatusCode::CANCELLED));
gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
ResetBackendCounters();
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);

47
test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
Unescape View File

@ -272,10 +272,14 @@ TEST_P(RingHashTest, HeaderHashing) {
const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options1);
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options2);
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options3);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options1);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options2);
@ -352,7 +356,8 @@ TEST_P(RingHashTest, NoHashPolicy) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -418,7 +423,8 @@ TEST_P(RingHashTest, HashOnHeaderThatIsNotPresent) {
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs, rpc_options);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -460,7 +466,8 @@ TEST_P(RingHashTest, UnsupportedHashPolicyDefaultToRandomHashing) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int request_count_1 = backends_[0]->backend_service()->request_count();
@ -496,7 +503,8 @@ TEST_P(RingHashTest, RandomHashingDistributionAccordingToEndpointWeight) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int weight_33_request_count =
@ -535,7 +543,8 @@ TEST_P(RingHashTest,
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// TODO(donnadionne): remove extended timeout after ring creation
// optimization.
WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_timeout_ms(kRpcTimeoutMs));
CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs);
const int weight_20_request_count =
@ -912,7 +921,8 @@ TEST_P(RingHashTest, TransientFailureCheckNextOne) {
{"address_hash",
CreateMetadataValueThatHashesToBackendPort(unused_port)}};
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
}
@ -940,12 +950,15 @@ TEST_P(RingHashTest, SwitchToLowerPrioirtyAndThenBack) {
std::vector<std::pair<std::string, std::string>> metadata = {
{"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
backends_[0]->StopListeningAndSendGoaways();
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
ShutdownBackend(0);
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackendOptions().set_allow_failures(true), rpc_options);
StartBackend(0);
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options);
EXPECT_EQ(100, backends_[0]->backend_service()->request_count());
EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
@ -1029,7 +1042,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) {
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
// RPCs should go to backend 0.
gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 0 ===");
WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
EXPECT_EQ(GRPC_CHANNEL_READY, channel_->GetState(false));
// Bring down backend 0 and bring up backend 1.
// Note the RPC contains a header value that will always be hashed to
@ -1059,7 +1073,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) {
EXPECT_TRUE(channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 1 ===");
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
gpr_log(GPR_INFO, "=== DONE ===");
}

8
test/cpp/end2end/xds/xds_rls_end2end_test.cc
Unescape View File

@ -153,8 +153,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPlugin) {
kRlsClusterSpecifierPluginInstanceName);
SetRouteConfiguration(balancer_.get(), new_route_config);
auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}});
WaitForAllBackends(DEBUG_LOCATION, 1, 2, WaitForBackendOptions(),
rpc_options);
WaitForAllBackends(DEBUG_LOCATION, 1, 2, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs, rpc_options);
// Make sure RPCs all go to the correct backend.
EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count());
@ -338,8 +338,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPluginDisabled) {
// Ensure we ignore the cluster specifier plugin and send traffic according to
// the default route.
auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}});
WaitForAllBackends(DEBUG_LOCATION, 0, 1, WaitForBackendOptions(),
rpc_options);
WaitForAllBackends(DEBUG_LOCATION, 0, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), rpc_options);
}
} // namespace

302
test/cpp/end2end/xds/xds_routing_end2end_test.cc
Unescape View File

@ -515,11 +515,15 @@ TEST_P(LdsRdsTest, ListenerRemoved) {
// Unset LDS resource.
balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
// Wait for RPCs to start failing.
do {
} while (SendRpc(RpcOptions(), nullptr).ok());
// Make sure RPCs are still failing.
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_times(1000));
SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) {
if (result.status.ok()) return true; // Keep going.
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(),
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"empty address list: ");
return false;
});
// Make sure we ACK'ed the update.
auto response_state = balancer_->ads_service()->lds_response_state();
ASSERT_TRUE(response_state.has_value());
@ -533,7 +537,13 @@ TEST_P(LdsRdsTest, NoMatchedDomain) {
route_config.mutable_virtual_hosts(0)->clear_domains();
route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
SetRouteConfiguration(balancer_.get(), route_config);
CheckRpcSendFailure(DEBUG_LOCATION);
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::UNAVAILABLE,
absl::StrCat(
(GetParam().enable_rds_testing() ? kDefaultRouteConfigurationName
: kServerName),
": UNAVAILABLE: could not find VirtualHost for ", kServerName,
" in RouteConfiguration"));
// Do a bit of polling, to allow the ACK to get to the ADS server.
channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100));
auto response_state = RouteConfigurationResponseState(balancer_.get());
@ -746,9 +756,8 @@ TEST_P(LdsRdsTest, MatchingRouteHasNoRouteAction) {
route->mutable_match()->set_prefix("");
route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), route_config);
CheckRpcSendFailure(DEBUG_LOCATION,
CheckRpcSendFailureOptions().set_expected_error_code(
StatusCode::UNAVAILABLE));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
"Matching route has inappropriate action");
}
TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) {
@ -1324,7 +1333,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForAllBackends(DEBUG_LOCATION, 0, 1);
WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs,
@ -1495,7 +1505,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForAllBackends(DEBUG_LOCATION, 0, 1);
WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(),
WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1626,7 +1637,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
default_route->mutable_route()->set_cluster(kDefaultClusterName);
SetRouteConfiguration(balancer_.get(), new_route_config);
WaitForBackend(DEBUG_LOCATION, 0);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1654,7 +1666,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
weighted_cluster2->mutable_weight()->set_value(kWeight50);
SetRouteConfiguration(balancer_.get(), new_route_config);
ResetBackendCounters();
WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs5050,
@ -1682,7 +1695,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
weighted_cluster2->mutable_weight()->set_value(kWeight25);
SetRouteConfiguration(balancer_.get(), new_route_config);
ResetBackendCounters();
WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525,
@ -1744,17 +1758,28 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) {
}
TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
CreateAndStartBackends(2);
// Start with only backend 1 up, but the default cluster pointing to
// backend 0, which is down.
CreateBackends(2);
StartBackend(1);
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
// Start an RPC with wait_for_ready=true and no deadline. This will
// stay pending until backend 0 is reachable.
LongRunningRpc rpc;
rpc.StartRpc(stub_.get(),
RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
// Send a non-wait_for_ready RPC, which should fail. This tells us
// that the client has received the update and attempted to connect.
constexpr char kErrorMessage[] =
// TODO(roth): Improve this error message as part of
// https://github.com/grpc/grpc/issues/22883.
"weighted_target: all children report state TRANSIENT_FAILURE";
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage);
// Now create a new cluster, pointing to backend 1.
const char* kNewClusterName = "new_cluster";
const char* kNewEdsServiceName = "new_eds_service_name";
// Populate new EDS resources.
EdsResourceArgs args({
{"locality0", CreateEndpointsForBackends(0, 1)},
});
EdsResourceArgs args1({
{"locality0", CreateEndpointsForBackends(1, 2)},
});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
EdsResourceArgs args1({{"locality0", CreateEndpointsForBackends(1, 2)}});
balancer_->ads_service()->SetEdsResource(
BuildEdsResource(args1, kNewEdsServiceName));
// Populate new CDS resources.
@ -1763,24 +1788,8 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
new_cluster.mutable_eds_cluster_config()->set_service_name(
kNewEdsServiceName);
balancer_->ads_service()->SetCdsResource(new_cluster);
// Bring down the current backend: 0, this will delay route picking time,
// resulting in un-committed RPCs.
ShutdownBackend(0);
// Send a RouteConfiguration with a default route that points to
// backend 0.
RouteConfiguration new_route_config = default_route_config_;
SetRouteConfiguration(balancer_.get(), new_route_config);
// Send exactly one RPC with no deadline and with wait_for_ready=true.
// This RPC will not complete until after backend 0 is started.
std::thread sending_rpc([this]() {
CheckRpcSendOk(DEBUG_LOCATION, 1,
RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
});
// Send a non-wait_for_ready RPC which should fail, this will tell us
// that the client has received the update and attempted to connect.
const Status status = SendRpc(RpcOptions().set_timeout_ms(0));
EXPECT_FALSE(status.ok());
// Send a update RouteConfiguration to use backend 1.
RouteConfiguration new_route_config = default_route_config_;
auto* default_route =
new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
default_route->mutable_route()->set_cluster(kNewClusterName);
@ -1789,13 +1798,20 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
// has processed the update.
WaitForBackend(
DEBUG_LOCATION, 1,
WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
true));
// Bring up the previous backend: 0, this will allow the delayed RPC to
// finally call on_call_committed upon completion.
[&](const RpcResult& result) {
if (!result.status.ok()) {
EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE);
EXPECT_EQ(result.status.error_message(), kErrorMessage);
}
},
WaitForBackendOptions().set_reset_counters(false));
// Bring up the backend 0. Yhis will allow the delayed RPC to finally
// complete.
StartBackend(0);
sending_rpc.join();
// Make sure RPCs go to the correct backend:
Status status = rpc.GetStatus();
EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
<< " message=" << status.error_message();
// Make sure RPCs went to the correct backends.
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
EXPECT_EQ(1, backends_[1]->backend_service()->request_count());
}
@ -1893,16 +1909,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(grpc_core::Duration::Seconds(
kTimeoutApplicationSecond)
.millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test max_stream_duration of 2.5 seconds applied
t0 = NowFromCycleCounter();
@ -1911,16 +1925,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(grpc_core::Duration::Seconds(
kTimeoutApplicationSecond)
.millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test http_stream_duration of 3.5 seconds applied
t0 = NowFromCycleCounter();
@ -1929,11 +1941,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
}
@ -2007,16 +2017,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
new_route_config);
// Test application timeout is applied for route 1
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2024,16 +2031,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
kTimeoutApplicationSecond * 1000000000);
// Test application timeout is applied for route 2
t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
system_clock::now() - t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
@ -2063,11 +2067,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
// Test application timeout is applied for route 1
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2084,11 +2086,9 @@ TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
@ -2115,52 +2115,34 @@ TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// Ensure we retried the correct number of times on all supported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::CANCELLED));
DEBUG_LOCATION, StatusCode::CANCELLED, "",
RpcOptions().set_server_expected_error(StatusCode::CANCELLED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::INTERNAL))
.set_expected_error_code(StatusCode::INTERNAL));
DEBUG_LOCATION, StatusCode::INTERNAL, "",
RpcOptions().set_server_expected_error(StatusCode::INTERNAL));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::RESOURCE_EXHAUSTED))
.set_expected_error_code(StatusCode::RESOURCE_EXHAUSTED));
DEBUG_LOCATION, StatusCode::RESOURCE_EXHAUSTED, "",
RpcOptions().set_server_expected_error(StatusCode::RESOURCE_EXHAUSTED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE))
.set_expected_error_code(StatusCode::UNAVAILABLE));
DEBUG_LOCATION, StatusCode::UNAVAILABLE, "",
RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
ResetBackendCounters();
// Ensure we don't retry on an unsupported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::UNAUTHENTICATED))
.set_expected_error_code(StatusCode::UNAUTHENTICATED));
DEBUG_LOCATION, StatusCode::UNAUTHENTICATED, "",
RpcOptions().set_server_expected_error(StatusCode::UNAUTHENTICATED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2182,11 +2164,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyAtVirtualHostLevel) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// Ensure we retried the correct number of times on a supported status.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
}
@ -2217,12 +2196,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
// No need to set max interval and just let it be the default of 10x of base.
// We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED));
EXPECT_EQ(1 + 1, backends_[0]->backend_service()->request_count());
}
@ -2260,12 +2236,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect 2 retry before the RPC times out with DEADLINE_EXCEEDED.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_timeout_ms(2500).set_server_expected_error(
StatusCode::CANCELLED));
EXPECT_EQ(2 + 1, backends_[0]->backend_service()->request_count());
}
@ -2286,11 +2259,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyUnsupportedStatusCode) {
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect no retry.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2319,11 +2289,8 @@ TEST_P(LdsRdsTest,
SetRouteConfiguration(balancer_.get(), new_route_config);
// We expect no retry.
CheckRpcSendFailure(
DEBUG_LOCATION,
CheckRpcSendFailureOptions()
.set_rpc_options(RpcOptions().set_server_expected_error(
StatusCode::DEADLINE_EXCEEDED))
.set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "",
RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED));
EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
}
@ -2447,8 +2414,8 @@ TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) {
.set_metadata(std::move(metadata));
// Make sure all backends are up.
WaitForBackend(DEBUG_LOCATION, 0);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
header_match_rpc_options);
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(), header_match_rpc_options);
// Send RPCs.
CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs);
CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs, header_match_rpc_options);
@ -2744,12 +2711,12 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
SetRouteConfiguration(balancer_.get(), route_config);
// Make sure all backends are up and that requests for each RPC
// service go to the right backends.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Requests for services Echo and Echo2 should have gone to backend 0.
@ -2764,16 +2731,17 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
// different RPC service, and wait for the client to make the change.
route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
SetRouteConfiguration(balancer_.get(), route_config);
WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(),
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions(),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Now repeat the earlier test, making sure all traffic goes to the
// right place.
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false));
WaitForBackend(DEBUG_LOCATION, 0,
WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO1));
WaitForBackend(DEBUG_LOCATION, 1,
WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr,
WaitForBackendOptions().set_reset_counters(false),
RpcOptions().set_rpc_service(SERVICE_ECHO2));
// Requests for services Echo and Echo1 should have gone to backend 0.

Write Preview
Loading…
Cancel
Save