From 8459824e5c598da59d78f94d7fc5649a54a7943c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 9 Jun 2022 18:00:38 -0700 Subject: [PATCH] xds end2end tests: change tests to check all RPC failure statuses (#29974) * WIP * fix from merge * fix ring_hash tests * ring hash: fix picker propagation bug in xds_cluster_manager policy * fix build * clang-format (ish) * fix build * more conversion * remove old CheckRpcSendFailure * fix SendRpcsAndCountFailuresWithMessage * fix WaitForBackend * clang-format * revert xds resolver change, that will be done in a separate PR * Automated change: Fix sanity tests Co-authored-by: markdroth --- .../end2end/xds/xds_cluster_end2end_test.cc | 180 ++++----- .../xds/xds_cluster_type_end2end_test.cc | 46 ++- test/cpp/end2end/xds/xds_core_end2end_test.cc | 131 ++++--- test/cpp/end2end/xds/xds_csds_end2end_test.cc | 48 ++- test/cpp/end2end/xds/xds_end2end_test.cc | 40 +- test/cpp/end2end/xds/xds_end2end_test_lib.cc | 122 +++--- test/cpp/end2end/xds/xds_end2end_test_lib.h | 67 +--- .../xds/xds_fault_injection_end2end_test.cc | 14 +- .../xds/xds_outlier_detection_end2end_test.cc | 366 ++++++++---------- .../end2end/xds/xds_ring_hash_end2end_test.cc | 47 ++- test/cpp/end2end/xds/xds_rls_end2end_test.cc | 8 +- .../end2end/xds/xds_routing_end2end_test.cc | 302 +++++++-------- 12 files changed, 672 insertions(+), 699 deletions(-) diff --git a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc index 037210d0e9f..f911a9612a6 100644 --- a/test/cpp/end2end/xds/xds_cluster_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_end2end_test.cc @@ -39,6 +39,7 @@ using ClientStats = LrsServiceImpl::ClientStats; constexpr char kLbDropType[] = "lb"; constexpr char kThrottleDropType[] = "throttle"; +constexpr char kStatusMessageDropPrefix[] = "EDS-configured drop: "; // // CDS tests @@ -239,11 +240,14 @@ TEST_P(CdsTest, ClusterRemoved) { // Unset CDS resource. balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); // Wait for RPCs to start failing. - do { - } while (SendRpc(RpcOptions(), nullptr).ok()); - // Make sure RPCs are still failing. - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_times(1000)); + SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) { + if (result.status.ok()) return true; // Keep going. + EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code()); + EXPECT_EQ(absl::StrCat("CDS resource \"", kDefaultClusterName, + "\" does not exist"), + result.status.error_message()); + return false; + }); // Make sure we ACK'ed the update. auto response_state = balancer_->ads_service()->cds_response_state(); ASSERT_TRUE(response_state.has_value()); @@ -349,23 +353,7 @@ TEST_P(CdsTest, ClusterChangeAfterAdsCallFails) { cluster.mutable_eds_cluster_config()->set_service_name(kNewEdsResourceName); balancer_->ads_service()->SetCdsResource(cluster); // Make sure client sees the change. - // TODO(roth): This should not be allowing errors. The errors are - // being caused by a bug that triggers in the following situation: - // - // 1. xDS call fails. - // 2. When xDS call is restarted, the server sends the updated CDS - // resource that points to the new EDS resource name. - // 3. When the client receives the CDS update, it does two things: - // - Sends the update to the CDS LB policy, which creates a new - // xds_cluster_resolver policy using the new EDS service name. - // - Notices that the CDS update no longer refers to the old EDS - // service name, so removes that resource, notifying the old - // xds_cluster_resolver policy that the resource no longer exists. - // - // Need to figure out a way to fix this bug, and then change this to - // not allow failures. - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true)); + WaitForBackend(DEBUG_LOCATION, 1); } // @@ -455,13 +443,21 @@ TEST_P(EdsTest, InitiallyEmptyServerlist) { EdsResourceArgs args({std::move(empty_locality)}); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // RPCs should fail. - CheckRpcSendFailure(DEBUG_LOCATION); + constexpr char kErrorMessage[] = + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"; + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage); // Send non-empty serverlist. args = EdsResourceArgs({{"locality0", CreateEndpointsForBackends()}}); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // RPCs should eventually succeed. - WaitForAllBackends(DEBUG_LOCATION, 0, 1, - WaitForBackendOptions().set_allow_failures(true)); + WaitForAllBackends(DEBUG_LOCATION, 0, 1, [&](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), kErrorMessage); + } + }); } // Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if @@ -493,17 +489,16 @@ TEST_P(EdsTest, BackendsRestart) { WaitForAllBackends(DEBUG_LOCATION); // Stop backends. RPCs should fail. ShutdownAllBackends(); - // Sending multiple failed requests instead of just one to ensure that the - // client notices that all backends are down before we restart them. If we - // didn't do this, then a single RPC could fail here due to the race - // condition between the LB pick and the GOAWAY from the chosen backend - // being shut down, which would not actually prove that the client noticed - // that all of the backends are down. Then, when we send another request - // below (which we expect to succeed), if the callbacks happen in the wrong - // order, the same race condition could happen again due to the client not - // yet having noticed that the backends were all down. - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_times(backends_.size())); + // Wait for channel to report TRANSIENT_FAILURE. + EXPECT_TRUE(channel_->WaitForStateChange( + GRPC_CHANNEL_READY, grpc_timeout_seconds_to_deadline(5))); + EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false)); + // RPCs should fail. + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"); // Restart all backends. RPCs should start succeeding again. StartAllBackends(); CheckRpcSendOk(DEBUG_LOCATION, 1, @@ -672,7 +667,7 @@ TEST_P(EdsTest, ManyLocalitiesStressTest) { } balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Wait until backend 0 is ready. - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); @@ -837,7 +832,8 @@ TEST_P(EdsTest, Drops) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Send kNumRpcs RPCs and count the drops. size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: "); + DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); // The drop rate should be roughly equal to the expectation. const double seen_drop_rate = static_cast(num_drops) / kNumRpcs; EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle, @@ -858,7 +854,8 @@ TEST_P(EdsTest, DropPerHundred) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Send kNumRpcs RPCs and count the drops. size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: "); + DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); // The drop rate should be roughly equal to the expectation. const double seen_drop_rate = static_cast(num_drops) / kNumRpcs; EXPECT_THAT(seen_drop_rate, @@ -879,7 +876,8 @@ TEST_P(EdsTest, DropPerTenThousand) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Send kNumRpcs RPCs and count the drops. size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: "); + DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); // The drop rate should be roughly equal to the expectation. const double seen_drop_rate = static_cast(num_drops) / kNumRpcs; EXPECT_THAT(seen_drop_rate, @@ -907,7 +905,8 @@ TEST_P(EdsTest, DropConfigUpdate) { // Send kNumRpcsLbOnly RPCs and count the drops. gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcsLbOnly, "EDS-configured drop: "); + DEBUG_LOCATION, kNumRpcsLbOnly, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); // The drop rate should be roughly equal to the expectation. double seen_drop_rate = static_cast(num_drops) / kNumRpcsLbOnly; @@ -924,24 +923,27 @@ TEST_P(EdsTest, DropConfigUpdate) { const double kDropRateThreshold = (kDropRateForLb + kDropRateForLbAndThrottle) / 2; size_t num_rpcs = kNumRpcsBoth; - while (seen_drop_rate < kDropRateThreshold) { - EchoResponse response; - const Status status = SendRpc(RpcOptions(), &response); - ++num_rpcs; - if (!status.ok() && - absl::StartsWith(status.error_message(), "EDS-configured drop: ")) { - ++num_drops; - } else { - EXPECT_TRUE(status.ok()) << "code=" << status.error_code() - << " message=" << status.error_message(); - EXPECT_EQ(response.message(), kRequestMessage); - } - seen_drop_rate = static_cast(num_drops) / num_rpcs; - } + SendRpcsUntil( + DEBUG_LOCATION, + [&](const RpcResult& result) { + ++num_rpcs; + if (result.status.ok()) { + EXPECT_EQ(result.response.message(), kRequestMessage); + } else { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_THAT(result.status.error_message(), + ::testing::StartsWith(kStatusMessageDropPrefix)); + ++num_drops; + } + seen_drop_rate = static_cast(num_drops) / num_rpcs; + return seen_drop_rate < kDropRateThreshold; + }, + /*timeout_ms=*/20000); // Send kNumRpcsBoth RPCs and count the drops. gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); num_drops = SendRpcsAndCountFailuresWithMessage(DEBUG_LOCATION, kNumRpcsBoth, - "EDS-configured drop: "); + StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); // The new drop rate should be roughly equal to the expectation. seen_drop_rate = static_cast(num_drops) / kNumRpcsBoth; @@ -962,7 +964,8 @@ TEST_P(EdsTest, DropAll) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Send kNumRpcs RPCs and all of them are dropped. size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "EDS-configured drop: "); + DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); EXPECT_EQ(num_drops, kNumRpcs); } @@ -997,7 +1000,7 @@ TEST_P(FailoverTest, ChooseHighestPriority) { 0}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 3, + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); for (size_t i = 0; i < 3; ++i) { EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); @@ -1017,7 +1020,7 @@ TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) { {"locality3", {}, kDefaultLocalityWeight, 0}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); for (size_t i = 1; i < 3; ++i) { EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); @@ -1049,7 +1052,7 @@ TEST_P(FailoverTest, Failover) { {"locality3", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 0}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); } @@ -1071,16 +1074,10 @@ TEST_P(FailoverTest, SwitchBackToHigherPriority) { }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); WaitForBackend(DEBUG_LOCATION, 3); - ShutdownBackend(3); + backends_[3]->StopListeningAndSendGoaways(); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1); ShutdownBackend(0); - WaitForBackend( - DEBUG_LOCATION, 1, - WaitForBackendOptions().set_reset_counters(false).set_allow_failures( - true)); - for (size_t i = 0; i < backends_.size(); ++i) { - if (i == 1) continue; - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } StartBackend(0); WaitForBackend(DEBUG_LOCATION, 0); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs); @@ -1096,7 +1093,11 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) { {"locality1", {MakeNonExistantEndpoint()}, kDefaultLocalityWeight, 1}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - CheckRpcSendFailure(DEBUG_LOCATION); + constexpr char kErrorMessage[] = + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"; + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage); args = EdsResourceArgs({ {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight, 0}, @@ -1104,8 +1105,12 @@ TEST_P(FailoverTest, UpdateInitialUnavailable) { 1}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 0, - WaitForBackendOptions().set_allow_failures(true)); + WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), kErrorMessage); + } + }); } // Tests that after the localities' priorities are updated, we still choose @@ -1124,7 +1129,7 @@ TEST_P(FailoverTest, UpdatePriority) { 0}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 3, + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); EXPECT_EQ(0U, backends_[0]->backend_service()->request_count()); EXPECT_EQ(0U, backends_[1]->backend_service()->request_count()); @@ -1161,7 +1166,7 @@ TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) { // When we get the first update, all backends in priority 0 are down, // so we will create priority 1. Backends 0 and 1 should have traffic, // but backend 2 should not. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count()); // Second update: @@ -1227,7 +1232,7 @@ TEST_P(FailoverTest, PriorityChildNameChurn) { 2}, }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - WaitForBackend(DEBUG_LOCATION, 3, + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); // P2 should not have gotten any traffic in this change. EXPECT_EQ(0UL, backends_[2]->backend_service()->request_count()); @@ -1255,14 +1260,15 @@ TEST_P(ClientLoadReportingTest, Vanilla) { }); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Wait until all backends are ready. - size_t num_warmup_rpcs = WaitForAllBackends( - DEBUG_LOCATION, 0, 4, WaitForBackendOptions().set_reset_counters(false)); + size_t num_warmup_rpcs = + WaitForAllBackends(DEBUG_LOCATION, 0, 4, /*check_status=*/nullptr, + WaitForBackendOptions().set_reset_counters(false)); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size()); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_times(kNumFailuresPerAddress * backends_.size()) - .set_rpc_options(RpcOptions().set_server_fail(true))); + for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) { + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "", + RpcOptions().set_server_fail(true)); + } const size_t total_successful_rpcs_sent = (kNumRpcsPerAddress * backends_.size()) + num_warmup_rpcs; const size_t total_failed_rpcs_sent = @@ -1321,10 +1327,10 @@ TEST_P(ClientLoadReportingTest, SendAllClusters) { size_t num_warmup_rpcs = WaitForAllBackends(DEBUG_LOCATION); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(DEBUG_LOCATION, kNumRpcsPerAddress * backends_.size()); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_times(kNumFailuresPerAddress * backends_.size()) - .set_rpc_options(RpcOptions().set_server_fail(true))); + for (size_t i = 0; i < kNumFailuresPerAddress * backends_.size(); ++i) { + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::FAILED_PRECONDITION, "", + RpcOptions().set_server_fail(true)); + } // Check that each backend got the right number of requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress, @@ -1552,7 +1558,6 @@ TEST_P(ClientLoadReportingTest, DropStats) { kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle; const size_t kNumRpcs = ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance); - const char kStatusMessageDropPrefix[] = "EDS-configured drop: "; // The ADS response contains two drop categories. EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); args.drop_categories = {{kLbDropType, kDropPerMillionForLb}, @@ -1560,7 +1565,8 @@ TEST_P(ClientLoadReportingTest, DropStats) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // Send kNumRpcs RPCs and count the drops. size_t num_drops = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, kStatusMessageDropPrefix); + DEBUG_LOCATION, kNumRpcs, StatusCode::UNAVAILABLE, + kStatusMessageDropPrefix); // The drop rate should be roughly equal to the expectation. const double seen_drop_rate = static_cast(num_drops) / kNumRpcs; EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle, diff --git a/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc b/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc index 71ab08727ce..bb303f5807c 100644 --- a/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_cluster_type_end2end_test.cc @@ -286,7 +286,7 @@ using AggregateClusterTest = ClusterTypeTest; INSTANTIATE_TEST_SUITE_P(XdsTest, AggregateClusterTest, ::testing::Values(XdsTestType()), &XdsTestType::Name); -TEST_P(AggregateClusterTest, ) { +TEST_P(AggregateClusterTest, Basic) { CreateAndStartBackends(2); const char* kNewCluster1Name = "new_cluster_1"; const char* kNewEdsService1Name = "new_eds_service_name_1"; @@ -326,13 +326,13 @@ TEST_P(AggregateClusterTest, ) { // Wait for traffic to go to backend 0. WaitForBackend(DEBUG_LOCATION, 0); // Shutdown backend 0 and wait for all traffic to go to backend 1. - ShutdownBackend(0); - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true)); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1); auto response_state = balancer_->ads_service()->cds_response_state(); ASSERT_TRUE(response_state.has_value()); EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED); - // Bring backend 0 back and ensure all traffic go back to it. + // Bring backend 0 back and ensure all traffic goes back to it. + ShutdownBackend(0); StartBackend(0); WaitForBackend(DEBUG_LOCATION, 0); } @@ -386,13 +386,13 @@ TEST_P(AggregateClusterTest, DiamondDependency) { // Wait for traffic to go to backend 0. WaitForBackend(DEBUG_LOCATION, 0); // Shutdown backend 0 and wait for all traffic to go to backend 1. - ShutdownBackend(0); - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true)); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1); auto response_state = balancer_->ads_service()->cds_response_state(); ASSERT_TRUE(response_state.has_value()); EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED); // Bring backend 0 back and ensure all traffic go back to it. + ShutdownBackend(0); StartBackend(0); WaitForBackend(DEBUG_LOCATION, 0); } @@ -522,8 +522,8 @@ TEST_P(AggregateClusterTest, FallBackWithConnectivityChurn) { connection_attempt_injector.Start(); // Wait for P0 backend. // Increase timeout to account for subchannel connection delays. - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), - RpcOptions().set_timeout_ms(2000)); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(2000)); // Send GOAWAY from the P0 backend. // We don't actually shut it down here to avoid flakiness caused by // failing an RPC after the client has already sent it but before the @@ -583,13 +583,13 @@ TEST_P(AggregateClusterTest, EdsToLogicalDns) { // Wait for traffic to go to backend 0. WaitForBackend(DEBUG_LOCATION, 0); // Shutdown backend 0 and wait for all traffic to go to backend 1. - ShutdownBackend(0); - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true)); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1); auto response_state = balancer_->ads_service()->cds_response_state(); ASSERT_TRUE(response_state.has_value()); EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED); // Bring backend 0 back and ensure all traffic go back to it. + ShutdownBackend(0); StartBackend(0); WaitForBackend(DEBUG_LOCATION, 0); } @@ -644,13 +644,13 @@ TEST_P(AggregateClusterTest, LogicalDnsToEds) { // Wait for traffic to go to backend 0. WaitForBackend(DEBUG_LOCATION, 0); // Shutdown backend 0 and wait for all traffic to go to backend 1. - ShutdownBackend(0); - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true)); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1); auto response_state = balancer_->ads_service()->cds_response_state(); ASSERT_TRUE(response_state.has_value()); EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED); // Bring backend 0 back and ensure all traffic go back to it. + ShutdownBackend(0); StartBackend(0); WaitForBackend(DEBUG_LOCATION, 0); } @@ -713,7 +713,11 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) { std::move(result)); } // When an RPC fails, we know the channel has seen the update. - CheckRpcSendFailure(DEBUG_LOCATION); + constexpr char kErrorMessage[] = + // TODO(roth): Figure out how to get some sort of resolution note + // included here as part of https://github.com/grpc/grpc/issues/22883. + "empty address list: "; + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage); // Send an EDS update that moves locality1 to priority 0. args1 = EdsResourceArgs({ {"locality1", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight, @@ -723,8 +727,12 @@ TEST_P(AggregateClusterTest, ReconfigEdsWhileLogicalDnsChildFails) { }); balancer_->ads_service()->SetEdsResource( BuildEdsResource(args1, kNewEdsService1Name)); - WaitForBackend(DEBUG_LOCATION, 0, - WaitForBackendOptions().set_allow_failures(true)); + WaitForBackend(DEBUG_LOCATION, 0, [&](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), kErrorMessage); + } + }); } TEST_P(AggregateClusterTest, MultipleClustersWithSameLocalities) { diff --git a/test/cpp/end2end/xds/xds_core_end2end_test.cc b/test/cpp/end2end/xds/xds_core_end2end_test.cc index 3588d080880..647c7d0e941 100644 --- a/test/cpp/end2end/xds/xds_core_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_core_end2end_test.cc @@ -198,9 +198,10 @@ TEST_P(XdsClientTest, MultipleBadCdsResources) { {"cluster", kClusterName2}, }; CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_metadata(std::move(metadata_cluster_2)))); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + "cluster_name_2: UNAVAILABLE: invalid resource: INVALID_ARGUMENT:.*" + "errors parsing CDS resource.*DiscoveryType is not valid.*", + RpcOptions().set_metadata(std::move(metadata_cluster_2))); } TEST_P(XdsClientTest, XdsStreamErrorPropagation) { @@ -394,16 +395,20 @@ INSTANTIATE_TEST_SUITE_P( TEST_P(TimeoutTest, LdsServerIgnoresRequest) { balancer_->ads_service()->IgnoreResourceType(kLdsTypeUrl); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, LdsResourceNotPresentInRequest) { balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) { @@ -417,8 +422,8 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) { CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000)); // Create second channel for a new server name. // This should fail because there is no LDS resource for this server name. - auto channel2 = - CreateChannel(/*failover_timeout_ms=*/0, "new-server.example.com"); + const char* kNewServerName = "new-server.example.com"; + auto channel2 = CreateChannel(/*failover_timeout_ms=*/0, kNewServerName); auto stub2 = grpc::testing::EchoTestService::NewStub(channel2); ClientContext context; EchoRequest request; @@ -428,21 +433,29 @@ TEST_P(TimeoutTest, LdsSecondResourceNotPresentInRequest) { auto status = SendRpcMethod(stub2.get(), rpc_options, &context, request, &response); EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code()); + EXPECT_THAT(status.error_message(), + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: "); } TEST_P(TimeoutTest, RdsServerIgnoresRequest) { balancer_->ads_service()->IgnoreResourceType(kRdsTypeUrl); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, RdsResourceNotPresentInRequest) { balancer_->ads_service()->UnsetResource(kRdsTypeUrl, kDefaultRouteConfigurationName); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) { @@ -456,12 +469,13 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) { CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_timeout_ms(4000)); // Add listener for 2nd channel, but no RDS resource. const char* kNewServerName = "new-server.example.com"; + const char* kNewRouteConfigName = "rds_resource_does_not_exist"; Listener listener = default_listener_; listener.set_name(kNewServerName); HttpConnectionManager http_connection_manager = ClientHcmAccessor().Unpack(listener); auto* rds = http_connection_manager.mutable_rds(); - rds->set_route_config_name("rds_resource_does_not_exist"); + rds->set_route_config_name(kNewRouteConfigName); rds->mutable_config_source()->mutable_self(); ClientHcmAccessor().Pack(http_connection_manager, &listener); balancer_->ads_service()->SetLdsResource(listener); @@ -478,20 +492,26 @@ TEST_P(TimeoutTest, RdsSecondResourceNotPresentInRequest) { auto status = SendRpcMethod(stub2.get(), rpc_options, &context, request, &response); EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code()); + EXPECT_EQ(status.error_message(), + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: "); } TEST_P(TimeoutTest, CdsServerIgnoresRequest) { balancer_->ads_service()->IgnoreResourceType(kCdsTypeUrl); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, CdsResourceNotPresentInRequest) { balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) { @@ -509,31 +529,38 @@ TEST_P(TimeoutTest, CdsSecondResourceNotPresentInRequest) { balancer_->ads_service()->SetRdsResource(route_config); // New cluster times out. // May need to wait a bit for the change to propagate to the client. - gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30); - bool error_seen = false; - do { - auto status = SendRpc(); - if (status.error_code() == StatusCode::UNAVAILABLE) { - error_seen = true; - break; - } - } while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); - EXPECT_TRUE(error_seen); + SendRpcsUntil( + DEBUG_LOCATION, + [&](const RpcResult& result) { + if (result.status.ok()) return true; // Keep going. + EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code()); + EXPECT_EQ(absl::StrCat("CDS resource \"", kNewClusterName, + "\" does not exist"), + result.status.error_message()); + return false; + }, + /*timeout_ms=*/30000, RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, EdsServerIgnoresRequest) { balancer_->ads_service()->IgnoreResourceType(kEdsTypeUrl); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE", + RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, EdsResourceNotPresentInRequest) { // No need to remove EDS resource, since the test suite does not add it // by default. - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_rpc_options( - RpcOptions().set_timeout_ms(4000))); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE", + RpcOptions().set_timeout_ms(4000)); } TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) { @@ -557,16 +584,20 @@ TEST_P(TimeoutTest, EdsSecondResourceNotPresentInRequest) { balancer_->ads_service()->SetRdsResource(route_config); // New EDS resource times out. // May need to wait a bit for the RDS change to propagate to the client. - gpr_timespec deadline = grpc_timeout_seconds_to_deadline(30); - bool error_seen = false; - do { - auto status = SendRpc(RpcOptions().set_rpc_method(METHOD_ECHO1)); - if (status.error_code() == StatusCode::UNAVAILABLE) { - error_seen = true; - break; - } - } while (gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0); - EXPECT_TRUE(error_seen); + SendRpcsUntil( + DEBUG_LOCATION, + [](const RpcResult& result) { + if (result.status.ok()) return true; // Keep going. + EXPECT_EQ(StatusCode::UNAVAILABLE, result.status.error_code()); + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + EXPECT_EQ( + result.status.error_message(), + "weighted_target: all children report state TRANSIENT_FAILURE"); + return false; + }, + /*timeout_ms=*/30000, + RpcOptions().set_rpc_method(METHOD_ECHO1).set_timeout_ms(4000)); } TEST_P(TimeoutTest, ServerDoesNotResendAfterAdsStreamRestart) { diff --git a/test/cpp/end2end/xds/xds_csds_end2end_test.cc b/test/cpp/end2end/xds/xds_csds_end2end_test.cc index 76c62a67176..364531018ec 100644 --- a/test/cpp/end2end/xds/xds_csds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_csds_end2end_test.cc @@ -560,11 +560,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) { TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) { int kTimeoutMillisecond = 1000; balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, + "Deadline Exceeded", + RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).generic_xds_configs(), ::testing::Contains(EqGenericXdsConfig( @@ -591,11 +589,9 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) { routes2->mutable_route()->set_cluster(kClusterName2); SetRouteConfiguration(balancer_.get(), route_config); // Try to get the configs plumb through - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, + "Deadline Exceeded", + RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).generic_xds_configs(), ::testing::AllOf( @@ -639,10 +635,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) { int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure. balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(grpc::StatusCode::UNAVAILABLE)); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).generic_xds_configs(), ::testing::Contains(EqGenericXdsConfig( @@ -656,10 +652,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) { balancer_->ads_service()->UnsetResource(kRdsTypeUrl, kDefaultRouteConfigurationName); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(grpc::StatusCode::UNAVAILABLE)); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: ", RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT( csds_response.config(0).generic_xds_configs(), @@ -672,10 +668,9 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) { int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure. balancer_->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(grpc::StatusCode::UNAVAILABLE)); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + absl::StrCat("CDS resource \"", kDefaultClusterName, "\" does not exist"), + RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).generic_xds_configs(), ::testing::Contains(EqGenericXdsConfig( @@ -687,10 +682,11 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) { int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure. balancer_->ads_service()->UnsetResource(kEdsTypeUrl, kDefaultEdsServiceName); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) - .set_expected_error_code(grpc::StatusCode::UNAVAILABLE)); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE", + RpcOptions().set_timeout_ms(kTimeoutMillisecond)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT( csds_response.config(0).generic_xds_configs(), diff --git a/test/cpp/end2end/xds/xds_end2end_test.cc b/test/cpp/end2end/xds/xds_end2end_test.cc index 7ec83290c62..ae1ec19fbdb 100644 --- a/test/cpp/end2end/xds/xds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_end2end_test.cc @@ -369,8 +369,16 @@ class XdsSecurityTest : public XdsEnd2endTest { continue; } } else { - WaitForBackend(DEBUG_LOCATION, 0, - WaitForBackendOptions().set_allow_failures(true)); + WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), + // TODO(roth): Improve this message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state " + "TRANSIENT_FAILURE"); + } + }); Status status = SendRpc(); if (!status.ok()) { gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.", @@ -768,11 +776,15 @@ TEST_P(XdsSecurityTest, TestTlsConfigurationInCombinedValidationContext) { ->set_instance_name("fake_plugin1"); transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancer_->ads_service()->SetCdsResource(cluster); - WaitForBackend(DEBUG_LOCATION, 0, - WaitForBackendOptions().set_allow_failures(true)); - Status status = SendRpc(); - EXPECT_TRUE(status.ok()) << "code=" << status.error_code() - << " message=" << status.error_message(); + WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), + // TODO(roth): Improve this message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"); + } + }); } // TODO(yashykt): Remove this test once we stop supporting old fields @@ -789,11 +801,15 @@ TEST_P(XdsSecurityTest, ->set_instance_name("fake_plugin1"); transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancer_->ads_service()->SetCdsResource(cluster); - WaitForBackend(DEBUG_LOCATION, 0, - WaitForBackendOptions().set_allow_failures(true)); - Status status = SendRpc(); - EXPECT_TRUE(status.ok()) << "code=" << status.error_code() - << " message=" << status.error_message(); + WaitForBackend(DEBUG_LOCATION, 0, [](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), + // TODO(roth): Improve this message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"); + } + }); } TEST_P(XdsSecurityTest, TestMtlsConfigurationWithNoSanMatchers) { diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index 3713099af50..8a539611f9d 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -834,20 +834,41 @@ Status XdsEnd2endTest::SendRpc(const RpcOptions& rpc_options, return status; } -void XdsEnd2endTest::CheckRpcSendOk( - const grpc_core::DebugLocation& debug_location, const size_t times, +void XdsEnd2endTest::SendRpcsUntil( + const grpc_core::DebugLocation& debug_location, + std::function continue_predicate, int timeout_ms, const RpcOptions& rpc_options) { - for (size_t i = 0; i < times; ++i) { - EchoResponse response; - const Status status = SendRpc(rpc_options, &response); - EXPECT_TRUE(status.ok()) - << "code=" << status.error_code() - << " message=" << status.error_message() << " at " + absl::Time deadline = absl::InfiniteFuture(); + if (timeout_ms != 0) { + deadline = absl::Now() + + (absl::Milliseconds(timeout_ms) * grpc_test_slowdown_factor()); + } + while (true) { + RpcResult result; + result.status = SendRpc(rpc_options, &result.response); + if (!continue_predicate(result)) return; + EXPECT_LE(absl::Now(), deadline) << debug_location.file() << ":" << debug_location.line(); - EXPECT_EQ(response.message(), kRequestMessage); + if (absl::Now() >= deadline) break; } } +void XdsEnd2endTest::CheckRpcSendOk( + const grpc_core::DebugLocation& debug_location, const size_t times, + const RpcOptions& rpc_options) { + SendRpcsUntil( + debug_location, + [debug_location, times, n = size_t(0)](const RpcResult& result) mutable { + EXPECT_TRUE(result.status.ok()) + << "code=" << result.status.error_code() + << " message=" << result.status.error_message() << " at " + << debug_location.file() << ":" << debug_location.line(); + EXPECT_EQ(result.response.message(), kRequestMessage); + return ++n < times; + }, + /*timeout_ms=*/0, rpc_options); +} + void XdsEnd2endTest::CheckRpcSendFailure( const grpc_core::DebugLocation& debug_location, StatusCode expected_status, absl::string_view expected_message_regex, const RpcOptions& rpc_options) { @@ -857,41 +878,29 @@ void XdsEnd2endTest::CheckRpcSendFailure( EXPECT_EQ(expected_status, status.error_code()) << debug_location.file() << ":" << debug_location.line(); EXPECT_THAT(status.error_message(), - ::testing::ContainsRegex(expected_message_regex)) + ::testing::MatchesRegex(expected_message_regex)) << debug_location.file() << ":" << debug_location.line(); } -void XdsEnd2endTest::CheckRpcSendFailure( - const grpc_core::DebugLocation& debug_location, - const CheckRpcSendFailureOptions& options) { - for (size_t i = 0; options.continue_predicate(i); ++i) { - const Status status = SendRpc(options.rpc_options); - EXPECT_FALSE(status.ok()) - << " at " << debug_location.file() << ":" << debug_location.line(); - if (options.expected_error_code != StatusCode::OK) { - EXPECT_EQ(options.expected_error_code, status.error_code()) - << "code=" << status.error_code() - << " message=" << status.error_message() << " at " - << debug_location.file() << ":" << debug_location.line(); - } - } -} - size_t XdsEnd2endTest::SendRpcsAndCountFailuresWithMessage( const grpc_core::DebugLocation& debug_location, size_t num_rpcs, - const char* drop_error_message_prefix, const RpcOptions& rpc_options) { + StatusCode expected_status, absl::string_view expected_message_prefix, + const RpcOptions& rpc_options) { size_t num_failed = 0; - for (size_t i = 0; i < num_rpcs; ++i) { - Status status = SendRpc(rpc_options); - if (!status.ok()) { - EXPECT_THAT(status.error_message(), - ::testing::StartsWith(drop_error_message_prefix)) - << "code=" << status.error_code() - << " message=" << status.error_message() << " at " - << debug_location.file() << ":" << debug_location.line(); - ++num_failed; - } - } + SendRpcsUntil( + debug_location, + [&, n = size_t(0)](const RpcResult& result) mutable { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), expected_status) + << debug_location.file() << ":" << debug_location.line(); + EXPECT_THAT(result.status.error_message(), + ::testing::StartsWith(expected_message_prefix)) + << debug_location.file() << ":" << debug_location.line(); + ++num_failed; + } + return ++n < num_rpcs; + }, + /*timeout_ms=*/0, rpc_options); return num_failed; } @@ -954,28 +963,29 @@ std::vector XdsEnd2endTest::SendConcurrentRpcs( size_t XdsEnd2endTest::WaitForAllBackends( const grpc_core::DebugLocation& debug_location, size_t start_index, - size_t stop_index, const WaitForBackendOptions& wait_options, - const RpcOptions& rpc_options) { - size_t num_rpcs = 0; - auto deadline = absl::Now() + (absl::Milliseconds(wait_options.timeout_ms) * - grpc_test_slowdown_factor()); + size_t stop_index, std::function check_status, + const WaitForBackendOptions& wait_options, const RpcOptions& rpc_options) { + if (check_status == nullptr) { + check_status = [&](const RpcResult& result) { + EXPECT_TRUE(result.status.ok()) + << "code=" << result.status.error_code() + << " message=" << result.status.error_message() << " at " + << debug_location.file() << ":" << debug_location.line(); + }; + } gpr_log(GPR_INFO, "========= WAITING FOR BACKENDS [%" PRIuPTR ", %" PRIuPTR ") ==========", start_index, stop_index); - while (!SeenAllBackends(start_index, stop_index, rpc_options.service)) { - Status status = SendRpc(rpc_options); - if (!wait_options.allow_failures) { - EXPECT_TRUE(status.ok()) - << "code=" << status.error_code() - << " message=" << status.error_message() << " at " - << debug_location.file() << ":" << debug_location.line(); - } - EXPECT_LE(absl::Now(), deadline) - << " at " << debug_location.file() << ":" << debug_location.line(); - if (absl::Now() >= deadline) break; - ++num_rpcs; - } + size_t num_rpcs = 0; + SendRpcsUntil( + debug_location, + [&](const RpcResult& result) { + ++num_rpcs; + check_status(result); + return !SeenAllBackends(start_index, stop_index, rpc_options.service); + }, + wait_options.timeout_ms, rpc_options); if (wait_options.reset_counters) ResetBackendCounters(); gpr_log(GPR_INFO, "Backends up; sent %" PRIuPTR " warm up requests", num_rpcs); diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index 45317b96ad4..b7ea4053c7b 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -801,6 +801,17 @@ class XdsEnd2endTest : public ::testing::TestWithParam { GPR_UNREACHABLE_CODE(return grpc::Status::OK); } + // Send RPCs in a loop until either continue_predicate() returns false + // or timeout_ms elapses. + struct RpcResult { + Status status; + EchoResponse response; + }; + void SendRpcsUntil(const grpc_core::DebugLocation& debug_location, + std::function continue_predicate, + int timeout_ms = 5000, + const RpcOptions& rpc_options = RpcOptions()); + // Sends the specified number of RPCs and fails if the RPC fails. void CheckRpcSendOk(const grpc_core::DebugLocation& debug_location, const size_t times = 1, @@ -813,49 +824,12 @@ class XdsEnd2endTest : public ::testing::TestWithParam { absl::string_view expected_message_regex, const RpcOptions& rpc_options = RpcOptions()); - // DEPRECATED -- USE THE ABOVE VARIANT INSTEAD. - // TODO(roth): Change all existing callers to use the above variant - // instead and then remove this. - struct CheckRpcSendFailureOptions { - std::function continue_predicate = [](size_t i) { - return i < 1; - }; - RpcOptions rpc_options; - StatusCode expected_error_code = StatusCode::OK; - - CheckRpcSendFailureOptions() {} - - CheckRpcSendFailureOptions& set_times(size_t times) { - continue_predicate = [times](size_t i) { return i < times; }; - return *this; - } - - CheckRpcSendFailureOptions& set_continue_predicate( - std::function pred) { - continue_predicate = std::move(pred); - return *this; - } - - CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) { - rpc_options = options; - return *this; - } - - CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) { - expected_error_code = code; - return *this; - } - }; - void CheckRpcSendFailure( - const grpc_core::DebugLocation& debug_location, - const CheckRpcSendFailureOptions& options = CheckRpcSendFailureOptions()); - // Sends num_rpcs RPCs, counting how many of them fail with a message - // matching the specfied drop_error_message_prefix. - // Any failure with a non-matching message is a test failure. + // matching the specfied expected_message_prefix. + // Any failure with a non-matching status or message is a test failure. size_t SendRpcsAndCountFailuresWithMessage( const grpc_core::DebugLocation& debug_location, size_t num_rpcs, - const char* drop_error_message_prefix, + StatusCode expected_status, absl::string_view expected_message_prefix, const RpcOptions& rpc_options = RpcOptions()); // A class for running a long-running RPC in its own thread. @@ -901,8 +875,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam { struct WaitForBackendOptions { // If true, resets the backend counters before returning. bool reset_counters = true; - // If true, RPC failures will not cause the test to fail. - bool allow_failures = false; // How long to wait for the backend(s) to see requests. int timeout_ms = 5000; @@ -913,11 +885,6 @@ class XdsEnd2endTest : public ::testing::TestWithParam { return *this; } - WaitForBackendOptions& set_allow_failures(bool enable) { - allow_failures = enable; - return *this; - } - WaitForBackendOptions& set_timeout_ms(int ms) { timeout_ms = ms; return *this; @@ -925,20 +892,24 @@ class XdsEnd2endTest : public ::testing::TestWithParam { }; // Sends RPCs until all of the backends in the specified range see requests. + // The check_status callback will be invoked to check the status of + // every RPC; if null, the default is to check that the RPC succeeded. // Returns the total number of RPCs sent. size_t WaitForAllBackends( const grpc_core::DebugLocation& debug_location, size_t start_index = 0, size_t stop_index = 0, + std::function check_status = nullptr, const WaitForBackendOptions& wait_options = WaitForBackendOptions(), const RpcOptions& rpc_options = RpcOptions()); // Sends RPCs until the backend at index backend_idx sees requests. void WaitForBackend( const grpc_core::DebugLocation& debug_location, size_t backend_idx, + std::function check_status = nullptr, const WaitForBackendOptions& wait_options = WaitForBackendOptions(), const RpcOptions& rpc_options = RpcOptions()) { WaitForAllBackends(debug_location, backend_idx, backend_idx + 1, - wait_options, rpc_options); + check_status, wait_options, rpc_options); } // diff --git a/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc index 0f5c983f82a..72700576f38 100644 --- a/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc @@ -116,12 +116,10 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) { // Config fault injection via different setup SetFilterConfig(http_fault); // Fire several RPCs, and expect all of them to be aborted. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_times(5) - .set_rpc_options(RpcOptions().set_wait_for_ready(true)) - .set_expected_error_code(StatusCode::ABORTED)); + for (size_t i = 0; i < 5; ++i) { + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::ABORTED, "Fault injected", + RpcOptions().set_wait_for_ready(true)); + } } // Without the listener config, the fault injection won't be enabled. @@ -166,7 +164,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) { SetFilterConfig(http_fault); // Send kNumRpcs RPCs and count the aborts. size_t num_aborted = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "Fault injected"); + DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected"); // The abort rate should be roughly equal to the expectation. const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; EXPECT_THAT(seen_abort_rate, @@ -196,7 +194,7 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) { {"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)}, }; size_t num_aborted = SendRpcsAndCountFailuresWithMessage( - DEBUG_LOCATION, kNumRpcs, "Fault injected", + DEBUG_LOCATION, kNumRpcs, StatusCode::ABORTED, "Fault injected", RpcOptions().set_metadata(metadata)); // The abort rate should be roughly equal to the expectation. const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; diff --git a/test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc b/test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc index 274ec1d15d5..e8c8bda8f9b 100644 --- a/test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc @@ -99,17 +99,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEjectionAndUnejection) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend is ejected, rpc destinated to it are now hashed to the other @@ -177,28 +175,24 @@ TEST_P(OutlierDetectionTest, SuccessRateMaxPercent) { const auto rpc_options1 = RpcOptions().set_metadata(metadata1); const auto rpc_options2 = RpcOptions().set_metadata(metadata2); const auto rpc_options3 = RpcOptions().set_metadata(metadata3); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options2); + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options3); // Cause 2 error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata1)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata1)) + .set_server_expected_error(StatusCode::CANCELLED)); CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options2); CheckRpcSendOk(DEBUG_LOCATION, 1, rpc_options3); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); @@ -280,17 +274,15 @@ TEST_P(OutlierDetectionTest, SuccessRateStdevFactor) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend experenced failure, but since the stdev_factor is high, no @@ -351,17 +343,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEnforcementPercentage) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend experenced failure, but since the enforcement percentage is 0, no @@ -421,17 +411,15 @@ TEST_P(OutlierDetectionTest, SuccessRateMinimumHosts) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // All traffic still reaching the original backends and no backends are @@ -492,17 +480,15 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // All traffic still reaching the original backends and no backends are @@ -567,18 +553,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEjectionAndUnejection) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend is ejected all traffic going to the ejected backend should now @@ -646,28 +630,24 @@ TEST_P(OutlierDetectionTest, FailurePercentageMaxPercentage) { const auto rpc_options1 = RpcOptions().set_metadata(metadata1); const auto rpc_options2 = RpcOptions().set_metadata(metadata2); const auto rpc_options3 = RpcOptions().set_metadata(metadata3); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options2); + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options3); // Cause 2 error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata1)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata1)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend should be ejected, trafficed picked up by another backend. @@ -743,18 +723,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageThreshold) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend experenced 1 failure, but since the threshold is 50 % no @@ -815,18 +793,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageEnforcementPercentage) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend experenced failure, but since the enforcement percentage is 0, no @@ -886,18 +862,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageMinimumHosts) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // All traffic still reaching the original backends and no backends are @@ -959,18 +933,16 @@ TEST_P(OutlierDetectionTest, FailurePercentageRequestVolume) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass to cause // the backend to be ejected. - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // All traffic still reaching the original backends and no backends are @@ -1050,10 +1022,14 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) { const auto rpc_options1 = RpcOptions().set_metadata(metadata1); const auto rpc_options2 = RpcOptions().set_metadata(metadata2); const auto rpc_options3 = RpcOptions().set_metadata(metadata3); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options2); + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options3); // Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1 // outlier detection interval to pass. The 2 errors to the 1 backend will make // exactly 1 outlier from the success rate algorithm; all 4 errors will make 3 @@ -1063,33 +1039,21 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) { // eject another backend because of failure percentage we will stop as we have // reached our 50% limit. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata1).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata1).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata2).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata2).set_server_expected_error( + StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options); @@ -1149,17 +1113,15 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentageBothDisabled) { {"address_hash", CreateMetadataValueThatHashesToBackend(1)}}; const auto rpc_options = RpcOptions().set_metadata(metadata); const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); // Cause an error and wait for 1 outlier detection interval to pass - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_metadata(std::move(metadata)) - .set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions() + .set_metadata(std::move(metadata)) + .set_server_expected_error(StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); // 1 backend experenced failure, but since there is no counting there is no @@ -1235,42 +1197,34 @@ TEST_P(OutlierDetectionTest, const auto rpc_options1 = RpcOptions().set_metadata(metadata1); const auto rpc_options2 = RpcOptions().set_metadata(metadata2); const auto rpc_options3 = RpcOptions().set_metadata(metadata3); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options2); + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options3); // Cause 2 errors on 1 backend and 1 error on 2 backends and wait for 1 // outlier detection interval to pass. The errors should have caused 2 // ejctionss but since the policy is disabled we are not ejecting any and // traffic flow as usual and RPCs reach destinated backends. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata1).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata1).set_server_expected_error( + StatusCode::CANCELLED)); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_metadata(metadata2).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_metadata(metadata2).set_server_expected_error( + StatusCode::CANCELLED)); gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100)); ResetBackendCounters(); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options); diff --git a/test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc b/test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc index c6b911b4679..c8bf07cf763 100644 --- a/test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc @@ -272,10 +272,14 @@ TEST_P(RingHashTest, HeaderHashing) { const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1)); const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2)); const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options1); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), rpc_options2); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), rpc_options3); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options1); + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options2); + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options3); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options1); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options2); @@ -352,7 +356,8 @@ TEST_P(RingHashTest, NoHashPolicy) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // TODO(donnadionne): remove extended timeout after ring creation // optimization. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs); const int request_count_1 = backends_[0]->backend_service()->request_count(); @@ -418,7 +423,8 @@ TEST_P(RingHashTest, HashOnHeaderThatIsNotPresent) { const auto rpc_options = RpcOptions().set_metadata(std::move(metadata)); // TODO(donnadionne): remove extended timeout after ring creation // optimization. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs, rpc_options); const int request_count_1 = backends_[0]->backend_service()->request_count(); @@ -460,7 +466,8 @@ TEST_P(RingHashTest, UnsupportedHashPolicyDefaultToRandomHashing) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // TODO(donnadionne): remove extended timeout after ring creation // optimization. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs); const int request_count_1 = backends_[0]->backend_service()->request_count(); @@ -496,7 +503,8 @@ TEST_P(RingHashTest, RandomHashingDistributionAccordingToEndpointWeight) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // TODO(donnadionne): remove extended timeout after ring creation // optimization. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs); const int weight_33_request_count = @@ -535,7 +543,8 @@ TEST_P(RingHashTest, balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); // TODO(donnadionne): remove extended timeout after ring creation // optimization. - WaitForAllBackends(DEBUG_LOCATION, 0, 2, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 0, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_timeout_ms(kRpcTimeoutMs)); CheckRpcSendOk(DEBUG_LOCATION, kNumRpcs); const int weight_20_request_count = @@ -912,7 +921,8 @@ TEST_P(RingHashTest, TransientFailureCheckNextOne) { {"address_hash", CreateMetadataValueThatHashesToBackendPort(unused_port)}}; const auto rpc_options = RpcOptions().set_metadata(std::move(metadata)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options); } @@ -940,12 +950,15 @@ TEST_P(RingHashTest, SwitchToLowerPrioirtyAndThenBack) { std::vector> metadata = { {"address_hash", CreateMetadataValueThatHashesToBackend(0)}}; const auto rpc_options = RpcOptions().set_metadata(std::move(metadata)); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); + backends_[0]->StopListeningAndSendGoaways(); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); ShutdownBackend(0); - WaitForBackend(DEBUG_LOCATION, 1, - WaitForBackendOptions().set_allow_failures(true), rpc_options); StartBackend(0); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); CheckRpcSendOk(DEBUG_LOCATION, 100, rpc_options); EXPECT_EQ(100, backends_[0]->backend_service()->request_count()); EXPECT_EQ(0, backends_[1]->backend_service()->request_count()); @@ -1029,7 +1042,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) { grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds))); // RPCs should go to backend 0. gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 0 ==="); - WaitForBackend(DEBUG_LOCATION, 0, WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); EXPECT_EQ(GRPC_CHANNEL_READY, channel_->GetState(false)); // Bring down backend 0 and bring up backend 1. // Note the RPC contains a header value that will always be hashed to @@ -1059,7 +1073,8 @@ TEST_P(RingHashTest, TransientFailureSkipToAvailableReady) { EXPECT_TRUE(channel_->WaitForConnected( grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds))); gpr_log(GPR_INFO, "=== WAITING FOR BACKEND 1 ==="); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); gpr_log(GPR_INFO, "=== DONE ==="); } diff --git a/test/cpp/end2end/xds/xds_rls_end2end_test.cc b/test/cpp/end2end/xds/xds_rls_end2end_test.cc index bd3f7e82f47..490e2dcf7fb 100644 --- a/test/cpp/end2end/xds/xds_rls_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_rls_end2end_test.cc @@ -153,8 +153,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPlugin) { kRlsClusterSpecifierPluginInstanceName); SetRouteConfiguration(balancer_.get(), new_route_config); auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}}); - WaitForAllBackends(DEBUG_LOCATION, 1, 2, WaitForBackendOptions(), - rpc_options); + WaitForAllBackends(DEBUG_LOCATION, 1, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs, rpc_options); // Make sure RPCs all go to the correct backend. EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count()); @@ -338,8 +338,8 @@ TEST_P(RlsTest, XdsRoutingClusterSpecifierPluginDisabled) { // Ensure we ignore the cluster specifier plugin and send traffic according to // the default route. auto rpc_options = RpcOptions().set_metadata({{kRlsTestKey1, kRlsTestValue}}); - WaitForAllBackends(DEBUG_LOCATION, 0, 1, WaitForBackendOptions(), - rpc_options); + WaitForAllBackends(DEBUG_LOCATION, 0, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), rpc_options); } } // namespace diff --git a/test/cpp/end2end/xds/xds_routing_end2end_test.cc b/test/cpp/end2end/xds/xds_routing_end2end_test.cc index bf705c4f4b1..b17be76e10f 100644 --- a/test/cpp/end2end/xds/xds_routing_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_routing_end2end_test.cc @@ -515,11 +515,15 @@ TEST_P(LdsRdsTest, ListenerRemoved) { // Unset LDS resource. balancer_->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); // Wait for RPCs to start failing. - do { - } while (SendRpc(RpcOptions(), nullptr).ok()); - // Make sure RPCs are still failing. - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_times(1000)); + SendRpcsUntil(DEBUG_LOCATION, [](const RpcResult& result) { + if (result.status.ok()) return true; // Keep going. + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "empty address list: "); + return false; + }); // Make sure we ACK'ed the update. auto response_state = balancer_->ads_service()->lds_response_state(); ASSERT_TRUE(response_state.has_value()); @@ -533,7 +537,13 @@ TEST_P(LdsRdsTest, NoMatchedDomain) { route_config.mutable_virtual_hosts(0)->clear_domains(); route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain"); SetRouteConfiguration(balancer_.get(), route_config); - CheckRpcSendFailure(DEBUG_LOCATION); + CheckRpcSendFailure( + DEBUG_LOCATION, StatusCode::UNAVAILABLE, + absl::StrCat( + (GetParam().enable_rds_testing() ? kDefaultRouteConfigurationName + : kServerName), + ": UNAVAILABLE: could not find VirtualHost for ", kServerName, + " in RouteConfiguration")); // Do a bit of polling, to allow the ACK to get to the ADS server. channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100)); auto response_state = RouteConfigurationResponseState(balancer_.get()); @@ -746,9 +756,8 @@ TEST_P(LdsRdsTest, MatchingRouteHasNoRouteAction) { route->mutable_match()->set_prefix(""); route->mutable_route()->set_cluster(kDefaultClusterName); SetRouteConfiguration(balancer_.get(), route_config); - CheckRpcSendFailure(DEBUG_LOCATION, - CheckRpcSendFailureOptions().set_expected_error_code( - StatusCode::UNAVAILABLE)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, + "Matching route has inappropriate action"); } TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) { @@ -1324,7 +1333,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) { default_route->mutable_route()->set_cluster(kDefaultClusterName); SetRouteConfiguration(balancer_.get(), new_route_config); WaitForAllBackends(DEBUG_LOCATION, 0, 1); - WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO1)); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs, @@ -1495,7 +1505,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) { default_route->mutable_route()->set_cluster(kDefaultClusterName); SetRouteConfiguration(balancer_.get(), new_route_config); WaitForAllBackends(DEBUG_LOCATION, 0, 1); - WaitForAllBackends(DEBUG_LOCATION, 1, 3, WaitForBackendOptions(), + WaitForAllBackends(DEBUG_LOCATION, 1, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO1)); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525, @@ -1626,7 +1637,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) { default_route->mutable_route()->set_cluster(kDefaultClusterName); SetRouteConfiguration(balancer_.get(), new_route_config); WaitForBackend(DEBUG_LOCATION, 0); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO1)); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525, @@ -1654,7 +1666,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) { weighted_cluster2->mutable_weight()->set_value(kWeight50); SetRouteConfiguration(balancer_.get(), new_route_config); ResetBackendCounters(); - WaitForBackend(DEBUG_LOCATION, 2, WaitForBackendOptions(), + WaitForBackend(DEBUG_LOCATION, 2, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO1)); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs5050, @@ -1682,7 +1695,8 @@ TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) { weighted_cluster2->mutable_weight()->set_value(kWeight25); SetRouteConfiguration(balancer_.get(), new_route_config); ResetBackendCounters(); - WaitForBackend(DEBUG_LOCATION, 3, WaitForBackendOptions(), + WaitForBackend(DEBUG_LOCATION, 3, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO1)); CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs7525, @@ -1744,17 +1758,28 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) { } TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) { - CreateAndStartBackends(2); + // Start with only backend 1 up, but the default cluster pointing to + // backend 0, which is down. + CreateBackends(2); + StartBackend(1); + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Start an RPC with wait_for_ready=true and no deadline. This will + // stay pending until backend 0 is reachable. + LongRunningRpc rpc; + rpc.StartRpc(stub_.get(), + RpcOptions().set_wait_for_ready(true).set_timeout_ms(0)); + // Send a non-wait_for_ready RPC, which should fail. This tells us + // that the client has received the update and attempted to connect. + constexpr char kErrorMessage[] = + // TODO(roth): Improve this error message as part of + // https://github.com/grpc/grpc/issues/22883. + "weighted_target: all children report state TRANSIENT_FAILURE"; + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE, kErrorMessage); + // Now create a new cluster, pointing to backend 1. const char* kNewClusterName = "new_cluster"; const char* kNewEdsServiceName = "new_eds_service_name"; - // Populate new EDS resources. - EdsResourceArgs args({ - {"locality0", CreateEndpointsForBackends(0, 1)}, - }); - EdsResourceArgs args1({ - {"locality0", CreateEndpointsForBackends(1, 2)}, - }); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + EdsResourceArgs args1({{"locality0", CreateEndpointsForBackends(1, 2)}}); balancer_->ads_service()->SetEdsResource( BuildEdsResource(args1, kNewEdsServiceName)); // Populate new CDS resources. @@ -1763,24 +1788,8 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) { new_cluster.mutable_eds_cluster_config()->set_service_name( kNewEdsServiceName); balancer_->ads_service()->SetCdsResource(new_cluster); - // Bring down the current backend: 0, this will delay route picking time, - // resulting in un-committed RPCs. - ShutdownBackend(0); - // Send a RouteConfiguration with a default route that points to - // backend 0. - RouteConfiguration new_route_config = default_route_config_; - SetRouteConfiguration(balancer_.get(), new_route_config); - // Send exactly one RPC with no deadline and with wait_for_ready=true. - // This RPC will not complete until after backend 0 is started. - std::thread sending_rpc([this]() { - CheckRpcSendOk(DEBUG_LOCATION, 1, - RpcOptions().set_wait_for_ready(true).set_timeout_ms(0)); - }); - // Send a non-wait_for_ready RPC which should fail, this will tell us - // that the client has received the update and attempted to connect. - const Status status = SendRpc(RpcOptions().set_timeout_ms(0)); - EXPECT_FALSE(status.ok()); // Send a update RouteConfiguration to use backend 1. + RouteConfiguration new_route_config = default_route_config_; auto* default_route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); default_route->mutable_route()->set_cluster(kNewClusterName); @@ -1789,13 +1798,20 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) { // has processed the update. WaitForBackend( DEBUG_LOCATION, 1, - WaitForBackendOptions().set_reset_counters(false).set_allow_failures( - true)); - // Bring up the previous backend: 0, this will allow the delayed RPC to - // finally call on_call_committed upon completion. + [&](const RpcResult& result) { + if (!result.status.ok()) { + EXPECT_EQ(result.status.error_code(), StatusCode::UNAVAILABLE); + EXPECT_EQ(result.status.error_message(), kErrorMessage); + } + }, + WaitForBackendOptions().set_reset_counters(false)); + // Bring up the backend 0. Yhis will allow the delayed RPC to finally + // complete. StartBackend(0); - sending_rpc.join(); - // Make sure RPCs go to the correct backend: + Status status = rpc.GetStatus(); + EXPECT_TRUE(status.ok()) << "code=" << status.error_code() + << " message=" << status.error_message(); + // Make sure RPCs went to the correct backends. EXPECT_EQ(1, backends_[0]->backend_service()->request_count()); EXPECT_EQ(1, backends_[1]->backend_service()->request_count()); } @@ -1893,16 +1909,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) + grpc_core::Duration::Milliseconds(kTimeoutMillis); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions() - .set_rpc_service(SERVICE_ECHO1) - .set_rpc_method(METHOD_ECHO1) - .set_wait_for_ready(true) - .set_timeout_ms(grpc_core::Duration::Seconds( - kTimeoutApplicationSecond) - .millis())) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions() + .set_rpc_service(SERVICE_ECHO1) + .set_rpc_method(METHOD_ECHO1) + .set_wait_for_ready(true) + .set_timeout_ms( + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) + .millis())); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); // Test max_stream_duration of 2.5 seconds applied t0 = NowFromCycleCounter(); @@ -1911,16 +1925,14 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { t2 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) + grpc_core::Duration::Milliseconds(kTimeoutMillis); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions() - .set_rpc_service(SERVICE_ECHO2) - .set_rpc_method(METHOD_ECHO2) - .set_wait_for_ready(true) - .set_timeout_ms(grpc_core::Duration::Seconds( - kTimeoutApplicationSecond) - .millis())) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions() + .set_rpc_service(SERVICE_ECHO2) + .set_rpc_method(METHOD_ECHO2) + .set_wait_for_ready(true) + .set_timeout_ms( + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) + .millis())); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); // Test http_stream_duration of 3.5 seconds applied t0 = NowFromCycleCounter(); @@ -1929,11 +1941,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { t2 = t0 + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) + grpc_core::Duration::Milliseconds(kTimeoutMillis); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( - grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions().set_wait_for_ready(true).set_timeout_ms( + grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); } @@ -2007,16 +2017,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) { new_route_config); // Test application timeout is applied for route 1 auto t0 = system_clock::now(); - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_rpc_service(SERVICE_ECHO1) - .set_rpc_method(METHOD_ECHO1) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, + "Deadline Exceeded", + RpcOptions() + .set_rpc_service(SERVICE_ECHO1) + .set_rpc_method(METHOD_ECHO1) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -2024,16 +2031,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) { kTimeoutApplicationSecond * 1000000000); // Test application timeout is applied for route 2 t0 = system_clock::now(); - CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions() - .set_rpc_service(SERVICE_ECHO2) - .set_rpc_method(METHOD_ECHO2) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, + "Deadline Exceeded", + RpcOptions() + .set_rpc_service(SERVICE_ECHO2) + .set_rpc_method(METHOD_ECHO2) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)); ellapsed_nano_seconds = std::chrono::duration_cast( system_clock::now() - t0); EXPECT_GT(ellapsed_nano_seconds.count(), @@ -2063,11 +2067,9 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) { // Test application timeout is applied for route 1 auto t0 = system_clock::now(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( - grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions().set_wait_for_ready(true).set_timeout_ms( + grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -2084,11 +2086,9 @@ TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) { balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); auto t0 = system_clock::now(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( - grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions().set_wait_for_ready(true).set_timeout_ms( + grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -2115,52 +2115,34 @@ TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) { SetRouteConfiguration(balancer_.get(), new_route_config); // Ensure we retried the correct number of times on all supported status. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_server_expected_error(StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::CANCELLED)); + DEBUG_LOCATION, StatusCode::CANCELLED, "", + RpcOptions().set_server_expected_error(StatusCode::CANCELLED)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); ResetBackendCounters(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::DEADLINE_EXCEEDED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "", + RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); ResetBackendCounters(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_server_expected_error(StatusCode::INTERNAL)) - .set_expected_error_code(StatusCode::INTERNAL)); + DEBUG_LOCATION, StatusCode::INTERNAL, "", + RpcOptions().set_server_expected_error(StatusCode::INTERNAL)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); ResetBackendCounters(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::RESOURCE_EXHAUSTED)) - .set_expected_error_code(StatusCode::RESOURCE_EXHAUSTED)); + DEBUG_LOCATION, StatusCode::RESOURCE_EXHAUSTED, "", + RpcOptions().set_server_expected_error(StatusCode::RESOURCE_EXHAUSTED)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); ResetBackendCounters(); CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE)) - .set_expected_error_code(StatusCode::UNAVAILABLE)); + DEBUG_LOCATION, StatusCode::UNAVAILABLE, "", + RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); ResetBackendCounters(); // Ensure we don't retry on an unsupported status. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::UNAUTHENTICATED)) - .set_expected_error_code(StatusCode::UNAUTHENTICATED)); + DEBUG_LOCATION, StatusCode::UNAUTHENTICATED, "", + RpcOptions().set_server_expected_error(StatusCode::UNAUTHENTICATED)); EXPECT_EQ(1, backends_[0]->backend_service()->request_count()); } @@ -2182,11 +2164,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyAtVirtualHostLevel) { SetRouteConfiguration(balancer_.get(), new_route_config); // Ensure we retried the correct number of times on a supported status. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::DEADLINE_EXCEEDED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "", + RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED)); EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count()); } @@ -2217,12 +2196,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) { // No need to set max interval and just let it be the default of 10x of base. // We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_timeout_ms(2500).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions().set_timeout_ms(2500).set_server_expected_error( + StatusCode::CANCELLED)); EXPECT_EQ(1 + 1, backends_[0]->backend_service()->request_count()); } @@ -2260,12 +2236,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) { SetRouteConfiguration(balancer_.get(), new_route_config); // We expect 2 retry before the RPC times out with DEADLINE_EXCEEDED. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options( - RpcOptions().set_timeout_ms(2500).set_server_expected_error( - StatusCode::CANCELLED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", + RpcOptions().set_timeout_ms(2500).set_server_expected_error( + StatusCode::CANCELLED)); EXPECT_EQ(2 + 1, backends_[0]->backend_service()->request_count()); } @@ -2286,11 +2259,8 @@ TEST_P(LdsRdsTest, XdsRetryPolicyUnsupportedStatusCode) { SetRouteConfiguration(balancer_.get(), new_route_config); // We expect no retry. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::DEADLINE_EXCEEDED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "", + RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED)); EXPECT_EQ(1, backends_[0]->backend_service()->request_count()); } @@ -2319,11 +2289,8 @@ TEST_P(LdsRdsTest, SetRouteConfiguration(balancer_.get(), new_route_config); // We expect no retry. CheckRpcSendFailure( - DEBUG_LOCATION, - CheckRpcSendFailureOptions() - .set_rpc_options(RpcOptions().set_server_expected_error( - StatusCode::DEADLINE_EXCEEDED)) - .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); + DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "", + RpcOptions().set_server_expected_error(StatusCode::DEADLINE_EXCEEDED)); EXPECT_EQ(1, backends_[0]->backend_service()->request_count()); } @@ -2447,8 +2414,8 @@ TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) { .set_metadata(std::move(metadata)); // Make sure all backends are up. WaitForBackend(DEBUG_LOCATION, 0); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), - header_match_rpc_options); + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), header_match_rpc_options); // Send RPCs. CheckRpcSendOk(DEBUG_LOCATION, kNumEchoRpcs); CheckRpcSendOk(DEBUG_LOCATION, kNumEcho1Rpcs, header_match_rpc_options); @@ -2744,12 +2711,12 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) { SetRouteConfiguration(balancer_.get(), route_config); // Make sure all backends are up and that requests for each RPC // service go to the right backends. - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); - WaitForBackend(DEBUG_LOCATION, 1, + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false), RpcOptions().set_rpc_service(SERVICE_ECHO1)); - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false), RpcOptions().set_rpc_service(SERVICE_ECHO2)); // Requests for services Echo and Echo2 should have gone to backend 0. @@ -2764,16 +2731,17 @@ TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) { // different RPC service, and wait for the client to make the change. route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/"); SetRouteConfiguration(balancer_.get(), route_config); - WaitForBackend(DEBUG_LOCATION, 1, WaitForBackendOptions(), + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, + WaitForBackendOptions(), RpcOptions().set_rpc_service(SERVICE_ECHO2)); // Now repeat the earlier test, making sure all traffic goes to the // right place. - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false)); - WaitForBackend(DEBUG_LOCATION, 0, + WaitForBackend(DEBUG_LOCATION, 0, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false), RpcOptions().set_rpc_service(SERVICE_ECHO1)); - WaitForBackend(DEBUG_LOCATION, 1, + WaitForBackend(DEBUG_LOCATION, 1, /*check_status=*/nullptr, WaitForBackendOptions().set_reset_counters(false), RpcOptions().set_rpc_service(SERVICE_ECHO2)); // Requests for services Echo and Echo1 should have gone to backend 0.