diff --git a/src/core/ext/xds/xds_client.cc b/src/core/ext/xds/xds_client.cc index f5ae4a61974..4715a231d69 100644 --- a/src/core/ext/xds/xds_client.cc +++ b/src/core/ext/xds/xds_client.cc @@ -275,6 +275,12 @@ class XdsClient::ChannelState::AdsCallState XdsApi::EdsUpdateMap eds_update_map) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + template + void RejectAdsUpdateLocked(grpc_millis update_time, + const XdsApi::AdsParseResult& result, + StateMap* state_map) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnRequestSent(void* arg, grpc_error_handle error); void OnRequestSentLocked(grpc_error_handle error) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); @@ -1145,6 +1151,47 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdateLocked( } } +namespace { + +// Update resource_metadata for NACK. +void UpdateResourceMetadataNacked(const std::string& version, + const std::string& details, + grpc_millis update_time, + XdsApi::ResourceMetadata* resource_metadata) { + resource_metadata->client_status = XdsApi::ResourceMetadata::NACKED; + resource_metadata->failed_version = version; + resource_metadata->failed_details = details; + resource_metadata->failed_update_time = update_time; +} + +} // namespace + +template +void XdsClient::ChannelState::AdsCallState::RejectAdsUpdateLocked( + grpc_millis update_time, const XdsApi::AdsParseResult& result, + StateMap* state_map) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] %s update NACKed containing %" PRIuPTR + " resources", + xds_client(), result.type_url.c_str(), + result.resource_names_failed.size()); + } + std::string details = grpc_error_std_string(result.parse_error); + for (auto& name : result.resource_names_failed) { + auto it = state_map->find(name); + if (it == state_map->end()) continue; + auto& state = it->second; + // Notify watchers of error. + for (const auto& p : state.watchers) { + p.first->OnError(GRPC_ERROR_REF(result.parse_error)); + } + // Update resource metadata for CSDS. + UpdateResourceMetadataNacked(result.version, details, update_time, + &state.meta); + } +} + void XdsClient::ChannelState::AdsCallState::OnRequestSent( void* arg, grpc_error_handle error) { AdsCallState* ads_calld = static_cast(arg); @@ -1223,10 +1270,6 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { state.nonce = std::move(result.nonce); // NACK or ACK the response. if (result.parse_error != GRPC_ERROR_NONE) { - xds_client()->UpdateResourceMetadataWithFailedParseResultLocked( - update_time, result); - GRPC_ERROR_UNREF(state.error); - state.error = result.parse_error; // NACK unacceptable update. gpr_log(GPR_ERROR, "[xds_client %p] ADS response invalid for resource type %s " @@ -1234,6 +1277,23 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { xds_client(), result.type_url.c_str(), result.version.c_str(), state.nonce.c_str(), grpc_error_std_string(result.parse_error).c_str()); + result.parse_error = + grpc_error_set_int(result.parse_error, GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNAVAILABLE); + GRPC_ERROR_UNREF(state.error); + state.error = result.parse_error; + if (result.type_url == XdsApi::kLdsTypeUrl) { + RejectAdsUpdateLocked(update_time, result, + &xds_client()->listener_map_); + } else if (result.type_url == XdsApi::kRdsTypeUrl) { + RejectAdsUpdateLocked(update_time, result, + &xds_client()->route_config_map_); + } else if (result.type_url == XdsApi::kCdsTypeUrl) { + RejectAdsUpdateLocked(update_time, result, &xds_client()->cluster_map_); + } else if (result.type_url == XdsApi::kEdsTypeUrl) { + RejectAdsUpdateLocked(update_time, result, + &xds_client()->endpoint_map_); + } SendMessageLocked(result.type_url); } else { seen_response_ = true; @@ -2253,44 +2313,6 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked( return snapshot_map; } -void XdsClient::UpdateResourceMetadataWithFailedParseResultLocked( - grpc_millis update_time, const XdsApi::AdsParseResult& result) { - // ADS update is rejected and the resource names in the failed update is - // available. - std::string details = grpc_error_std_string(result.parse_error); - for (auto& name : result.resource_names_failed) { - XdsApi::ResourceMetadata* resource_metadata = nullptr; - if (result.type_url == XdsApi::kLdsTypeUrl) { - auto it = listener_map_.find(name); - if (it != listener_map_.end()) { - resource_metadata = &it->second.meta; - } - } else if (result.type_url == XdsApi::kRdsTypeUrl) { - auto it = route_config_map_.find(name); - if (route_config_map_.find(name) != route_config_map_.end()) { - resource_metadata = &it->second.meta; - } - } else if (result.type_url == XdsApi::kCdsTypeUrl) { - auto it = cluster_map_.find(name); - if (cluster_map_.find(name) != cluster_map_.end()) { - resource_metadata = &it->second.meta; - } - } else if (result.type_url == XdsApi::kEdsTypeUrl) { - auto it = endpoint_map_.find(name); - if (endpoint_map_.find(name) != endpoint_map_.end()) { - resource_metadata = &it->second.meta; - } - } - if (resource_metadata == nullptr) { - return; - } - resource_metadata->client_status = XdsApi::ResourceMetadata::NACKED; - resource_metadata->failed_version = result.version; - resource_metadata->failed_details = details; - resource_metadata->failed_update_time = update_time; - } -} - std::string XdsClient::DumpClientConfigBinary() { MutexLock lock(&mu_); XdsApi::ResourceTypeMetadataMap resource_type_metadata_map; diff --git a/src/core/ext/xds/xds_client.h b/src/core/ext/xds/xds_client.h index 0ee84e7866d..27796f412aa 100644 --- a/src/core/ext/xds/xds_client.h +++ b/src/core/ext/xds/xds_client.h @@ -324,10 +324,6 @@ class XdsClient : public DualRefCounted { bool send_all_clusters, const std::set& clusters) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - void UpdateResourceMetadataWithFailedParseResultLocked( - grpc_millis update_time, const XdsApi::AdsParseResult& result) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - std::unique_ptr bootstrap_; grpc_channel_args* args_; const grpc_millis request_timeout_; diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index f4c9d90d6f1..cd2ce0fb113 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -31,6 +31,7 @@ #include "absl/functional/bind_front.h" #include "absl/memory/memory.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" @@ -2216,18 +2217,91 @@ class XdsEnd2endTest : public ::testing::TestWithParam { } } - void CheckRpcSendFailure( - const size_t times = 1, const RpcOptions& rpc_options = RpcOptions(), - const StatusCode expected_error_code = StatusCode::OK) { - for (size_t i = 0; i < times; ++i) { - const Status status = SendRpc(rpc_options); + struct CheckRpcSendFailureOptions { + std::function continue_predicate = [](size_t i) { + return i < 1; + }; + RpcOptions rpc_options; + StatusCode expected_error_code = StatusCode::OK; + + CheckRpcSendFailureOptions() {} + + CheckRpcSendFailureOptions& set_times(size_t times) { + continue_predicate = [times](size_t i) { return i < times; }; + return *this; + } + + CheckRpcSendFailureOptions& set_continue_predicate( + std::function pred) { + continue_predicate = std::move(pred); + return *this; + } + + CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) { + rpc_options = options; + return *this; + } + + CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) { + expected_error_code = code; + return *this; + } + }; + + void CheckRpcSendFailure(const CheckRpcSendFailureOptions& options = + CheckRpcSendFailureOptions()) { + for (size_t i = 0; options.continue_predicate(i); ++i) { + const Status status = SendRpc(options.rpc_options); EXPECT_FALSE(status.ok()); - if (expected_error_code != StatusCode::OK) { - EXPECT_EQ(expected_error_code, status.error_code()); + if (options.expected_error_code != StatusCode::OK) { + EXPECT_EQ(options.expected_error_code, status.error_code()); } } } + bool WaitForNack( + std::function get_state, + StatusCode expected_status = StatusCode::UNAVAILABLE) { + auto deadline = absl::Now() + absl::Seconds(30); + bool success = true; + CheckRpcSendFailure(CheckRpcSendFailureOptions() + .set_continue_predicate([&](size_t) { + if (absl::Now() >= deadline) { + success = false; + return false; + } + return get_state() != + AdsServiceImpl::ResponseState::NACKED; + }) + .set_expected_error_code(expected_status)); + return success; + } + + bool WaitForLdsNack(StatusCode expected_status = StatusCode::UNAVAILABLE) { + return WaitForNack( + [&]() { + return balancers_[0]->ads_service()->lds_response_state().state; + }, + expected_status); + } + + bool WaitForRdsNack() { + return WaitForNack( + [&]() { return RouteConfigurationResponseState(0).state; }); + } + + bool WaitForCdsNack() { + return WaitForNack([&]() { + return balancers_[0]->ads_service()->cds_response_state().state; + }); + } + + bool WaitForEdsNack() { + return WaitForNack([&]() { + return balancers_[0]->ads_service()->eds_response_state().state; + }); + } + static Listener BuildListener(const RouteConfiguration& route_config) { HttpConnectionManager http_connection_manager; *(http_connection_manager.mutable_route_config()) = route_config; @@ -2891,7 +2965,7 @@ TEST_P(BasicTest, BackendsRestart) { // below (which we expect to succeed), if the callbacks happen in the wrong // order, the same race condition could happen again due to the client not // yet having noticed that the backends were all down. - CheckRpcSendFailure(num_backends_); + CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(num_backends_)); // Restart all backends. RPCs should start succeeding again. StartAllBackends(); CheckRpcSendOk(1, RpcOptions().set_timeout_ms(2000).set_wait_for_ready(true)); @@ -3008,7 +3082,7 @@ TEST_P(XdsResolverOnlyTest, ClusterRemoved) { do { } while (SendRpc(RpcOptions(), nullptr).ok()); // Make sure RPCs are still failing. - CheckRpcSendFailure(1000); + CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000)); // Make sure we ACK'ed the update. EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state, AdsServiceImpl::ResponseState::ACKED); @@ -3276,17 +3350,30 @@ TEST_P(GlobalXdsClientTest, MultipleBadResources) { EchoResponse response; grpc::Status status = stub2->Echo(&context, request, &response); EXPECT_FALSE(status.ok()); - const auto response_state = - balancers_[0]->ads_service()->lds_response_state(); - EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); - EXPECT_THAT( - response_state.error_message, - ::testing::AllOf( - ::testing::HasSubstr(absl::StrCat( - kServerName, ": Listener has neither address nor ApiListener")), - ::testing::HasSubstr( - absl::StrCat(kServerName2, - ": Listener has neither address nor ApiListener")))); + // Wait for second NACK to be reported to xDS server. + auto deadline = absl::Now() + absl::Seconds(30); + bool timed_out = false; + CheckRpcSendFailure( + CheckRpcSendFailureOptions().set_continue_predicate([&](size_t) { + if (absl::Now() >= deadline) { + timed_out = true; + return false; + } + const auto response_state = + balancers_[0]->ads_service()->lds_response_state(); + return response_state.state != AdsServiceImpl::ResponseState::NACKED || + !absl::StrContains( + response_state.error_message, + absl::StrCat( + kServerName, + ": Listener has neither address nor ApiListener")) || + !absl::StrContains( + response_state.error_message, + absl::StrCat( + kServerName2, + ": Listener has neither address nor ApiListener")); + })); + ASSERT_FALSE(timed_out); } class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest { @@ -3462,7 +3549,7 @@ TEST_P(LdsTest, NoApiListener) { balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3484,7 +3571,7 @@ TEST_P(LdsTest, WrongRouteSpecifier) { balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3508,7 +3595,7 @@ TEST_P(LdsTest, RdsMissingConfigSource) { balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3533,7 +3620,7 @@ TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) { balancers_[0]->ads_service()->SetLdsResource(listener); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3606,11 +3693,7 @@ TEST_P(LdsTest, RejectsEmptyHttpFilterName) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3631,11 +3714,7 @@ TEST_P(LdsTest, RejectsDuplicateHttpFilterName) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3657,11 +3736,7 @@ TEST_P(LdsTest, RejectsUnknownHttpFilterType) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3707,11 +3782,7 @@ TEST_P(LdsTest, RejectsHttpFilterWithoutConfig) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3759,11 +3830,7 @@ TEST_P(LdsTest, RejectsUnparseableHttpFilterType) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3789,11 +3856,7 @@ TEST_P(LdsTest, RejectsHttpFiltersNotSupportedOnClients) { SetListenerAndRouteConfiguration(0, listener, default_route_config_); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -3888,7 +3951,7 @@ TEST_P(LdsRdsTest, ListenerRemoved) { do { } while (SendRpc(RpcOptions(), nullptr).ok()); // Make sure RPCs are still failing. - CheckRpcSendFailure(1000); + CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000)); // Make sure we ACK'ed the update. EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state, AdsServiceImpl::ResponseState::ACKED); @@ -3952,7 +4015,7 @@ TEST_P(LdsRdsTest, RouteMatchHasQueryParameters) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -3985,7 +4048,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4001,7 +4064,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4017,7 +4080,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixDoubleSlash) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4033,7 +4096,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4049,7 +4112,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4065,7 +4128,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathTooManySlashes) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4081,7 +4144,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathOnlyOneSlash) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4097,7 +4160,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4113,7 +4176,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4130,7 +4193,7 @@ TEST_P(LdsRdsTest, RouteMatchHasInvalidPathRegex) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4146,7 +4209,7 @@ TEST_P(LdsRdsTest, RouteHasNoRouteAction) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4164,7 +4227,7 @@ TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -4192,7 +4255,7 @@ TEST_P(LdsRdsTest, RouteActionWeightedTargetHasIncorrectTotalWeightSet) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4219,7 +4282,7 @@ TEST_P(LdsRdsTest, RouteActionWeightedClusterHasZeroTotalWeight) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -4247,7 +4310,7 @@ TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasEmptyClusterName) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4274,7 +4337,7 @@ TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasNoWeight) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -4294,7 +4357,7 @@ TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRegex) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -4316,7 +4379,7 @@ TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRange) { SetRouteConfiguration(0, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -5316,13 +5379,15 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { grpc_millis t1 = t0 + kTimeoutGrpcTimeoutHeaderMaxSecond * 1000 + kTimeoutMillis; grpc_millis t2 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis; - CheckRpcSendFailure(1, - RpcOptions() - .set_rpc_service(SERVICE_ECHO1) - .set_rpc_method(METHOD_ECHO1) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options( + RpcOptions() + .set_rpc_service(SERVICE_ECHO1) + .set_rpc_method(METHOD_ECHO1) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); t0 = NowFromCycleCounter(); EXPECT_GE(t0, t1); EXPECT_LT(t0, t2); @@ -5330,13 +5395,15 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { t0 = NowFromCycleCounter(); t1 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis; t2 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis; - CheckRpcSendFailure(1, - RpcOptions() - .set_rpc_service(SERVICE_ECHO2) - .set_rpc_method(METHOD_ECHO2) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options( + RpcOptions() + .set_rpc_service(SERVICE_ECHO2) + .set_rpc_method(METHOD_ECHO2) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); t0 = NowFromCycleCounter(); EXPECT_GE(t0, t1); EXPECT_LT(t0, t2); @@ -5344,10 +5411,11 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { t0 = NowFromCycleCounter(); t1 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis; t2 = t0 + kTimeoutApplicationSecond * 1000 + kTimeoutMillis; - CheckRpcSendFailure(1, - RpcOptions().set_wait_for_ready(true).set_timeout_ms( - kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( + kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); t0 = NowFromCycleCounter(); EXPECT_GE(t0, t1); EXPECT_LT(t0, t2); @@ -5427,13 +5495,15 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) { SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config); // Test application timeout is applied for route 1 auto t0 = system_clock::now(); - CheckRpcSendFailure(1, - RpcOptions() - .set_rpc_service(SERVICE_ECHO1) - .set_rpc_method(METHOD_ECHO1) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options( + RpcOptions() + .set_rpc_service(SERVICE_ECHO1) + .set_rpc_method(METHOD_ECHO1) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -5441,13 +5511,15 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) { kTimeoutApplicationSecond * 1000000000); // Test application timeout is applied for route 2 t0 = system_clock::now(); - CheckRpcSendFailure(1, - RpcOptions() - .set_rpc_service(SERVICE_ECHO2) - .set_rpc_method(METHOD_ECHO2) - .set_wait_for_ready(true) - .set_timeout_ms(kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options( + RpcOptions() + .set_rpc_service(SERVICE_ECHO2) + .set_rpc_method(METHOD_ECHO2) + .set_wait_for_ready(true) + .set_timeout_ms(kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); ellapsed_nano_seconds = std::chrono::duration_cast( system_clock::now() - t0); EXPECT_GT(ellapsed_nano_seconds.count(), @@ -5479,10 +5551,11 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) { default_route_config_); // Test application timeout is applied for route 1 auto t0 = system_clock::now(); - CheckRpcSendFailure(1, - RpcOptions().set_wait_for_ready(true).set_timeout_ms( - kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( + kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -5501,10 +5574,11 @@ TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) { {{"locality0", {MakeNonExistantEndpoint()}}}); balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args)); auto t0 = system_clock::now(); - CheckRpcSendFailure(1, - RpcOptions().set_wait_for_ready(true).set_timeout_ms( - kTimeoutApplicationSecond * 1000), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms( + kTimeoutApplicationSecond * 1000)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); auto ellapsed_nano_seconds = std::chrono::duration_cast(system_clock::now() - t0); @@ -5922,11 +5996,7 @@ TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInVirtualHost) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -5967,11 +6037,7 @@ TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInVirtualHost) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -5990,11 +6056,7 @@ TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInVirtualHost) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6035,11 +6097,7 @@ TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInVirtualHost) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -6058,11 +6116,7 @@ TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInRoute) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6105,11 +6159,7 @@ TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInRoute) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6129,11 +6179,7 @@ TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInRoute) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6176,11 +6222,7 @@ TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInRoute) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -6204,11 +6246,7 @@ TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInClusterWeight) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6261,11 +6299,7 @@ TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInClusterWeight) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6291,11 +6325,7 @@ TEST_P(LdsRdsTest, SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT(response_state.error_message, @@ -6348,11 +6378,7 @@ TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInClusterWeight) { SetListenerAndRouteConfiguration(0, default_listener_, route_config); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (RouteConfigurationResponseState(0).state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK"; const auto response_state = RouteConfigurationResponseState(0); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); EXPECT_THAT( @@ -6411,11 +6437,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeMissingLoadAssignment) { auto cluster = default_cluster_; cluster.set_type(Cluster::LOGICAL_DNS); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6436,11 +6458,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeMissingLocalities) { cluster.set_type(Cluster::LOGICAL_DNS); cluster.mutable_load_assignment(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6464,11 +6482,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeMultipleLocalities) { load_assignment->add_endpoints(); load_assignment->add_endpoints(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6490,11 +6504,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeMissingEndpoints) { cluster.set_type(Cluster::LOGICAL_DNS); cluster.mutable_load_assignment()->add_endpoints(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6518,11 +6528,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeMultipleEndpoints) { locality->add_lb_endpoints(); locality->add_lb_endpoints(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6544,11 +6550,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeEmptyEndpoint) { cluster.set_type(Cluster::LOGICAL_DNS); cluster.mutable_load_assignment()->add_endpoints()->add_lb_endpoints(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6571,11 +6573,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeEndpointMissingAddress) { ->add_lb_endpoints() ->mutable_endpoint(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6599,11 +6597,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeAddressMissingSocketAddress) { ->mutable_endpoint() ->mutable_address(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6629,11 +6623,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressHasResolverName) { ->mutable_socket_address() ->set_resolver_name("foo"); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6659,11 +6649,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingAddress) { ->mutable_address() ->mutable_socket_address(); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6689,11 +6675,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingPort) { ->mutable_socket_address() ->set_address(kServerName); balancers_[0]->ads_service()->SetCdsResource(cluster); - // Wait until xDS server sees NACK. - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->cds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6893,7 +6875,7 @@ TEST_P(CdsTest, LogicalDNSClusterTypeDisabled) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6915,7 +6897,7 @@ TEST_P(CdsTest, AggregateClusterTypeDisabled) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6931,7 +6913,7 @@ TEST_P(CdsTest, UnsupportedClusterType) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6958,7 +6940,7 @@ TEST_P(CdsTest, MultipleBadResources) { // Send RPC. SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6978,7 +6960,7 @@ TEST_P(CdsTest, WrongEdsConfig) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -6994,7 +6976,7 @@ TEST_P(CdsTest, WrongLbPolicy) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7010,7 +6992,7 @@ TEST_P(CdsTest, WrongLrsServer) { balancers_[0]->ads_service()->SetCdsResource(cluster); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7586,10 +7568,10 @@ TEST_P(CdsTest, RingHashAllFailReattempt) { SetNextResolutionForLbChannelAllBalancers(); std::vector> metadata = { {"address_hash", CreateMetadataValueThatHashesToBackend(0)}}; - const auto rpc_options = RpcOptions().set_metadata(std::move(metadata)); EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false)); ShutdownBackend(1); - CheckRpcSendFailure(1, rpc_options); + CheckRpcSendFailure(CheckRpcSendFailureOptions().set_rpc_options( + RpcOptions().set_metadata(std::move(metadata)))); StartBackend(1); // Ensure we are actively connecting without any traffic. EXPECT_TRUE(channel_->WaitForConnected( @@ -7628,7 +7610,8 @@ TEST_P(CdsTest, RingHashTransientFailureSkipToAvailableReady) { EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false)); ShutdownBackend(0); ShutdownBackend(1); - CheckRpcSendFailure(1, rpc_options); + CheckRpcSendFailure( + CheckRpcSendFailureOptions().set_rpc_options(rpc_options)); EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false)); // Bring up 0, should be picked as the RPC is hashed to it. StartBackend(0); @@ -7646,7 +7629,8 @@ TEST_P(CdsTest, RingHashTransientFailureSkipToAvailableReady) { // unused ports to fill the ring, it is almost guaranteed that the Picker // will go through some non-READY entries and skip them as per design. ShutdownBackend(0); - CheckRpcSendFailure(1, rpc_options); + CheckRpcSendFailure( + CheckRpcSendFailureOptions().set_rpc_options(rpc_options)); StartBackend(1); EXPECT_TRUE(channel_->WaitForConnected( grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds))); @@ -7714,7 +7698,7 @@ TEST_P(CdsTest, RingHashPolicyHasInvalidHashFunction) { balancers_[0]->ads_service()->SetEdsResource( BuildEdsResource(args, DefaultEdsServiceName())); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7743,7 +7727,7 @@ TEST_P(CdsTest, RingHashPolicyHasInvalidMinimumRingSize) { balancers_[0]->ads_service()->SetEdsResource( BuildEdsResource(args, DefaultEdsServiceName())); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7772,7 +7756,7 @@ TEST_P(CdsTest, RingHashPolicyHasInvalidMaxmumRingSize) { balancers_[0]->ads_service()->SetEdsResource( BuildEdsResource(args, DefaultEdsServiceName())); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7803,7 +7787,7 @@ TEST_P(CdsTest, RingHashPolicyHasInvalidRingSizeMinGreaterThanMax) { balancers_[0]->ads_service()->SetEdsResource( BuildEdsResource(args, DefaultEdsServiceName())); SetNextResolutionForLbChannelAllBalancers(); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7972,7 +7956,7 @@ TEST_P(XdsSecurityTest, UnknownTransportSocket) { auto* transport_socket = cluster.mutable_transport_socket(); transport_socket->set_name("unknown_transport_socket"); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -7987,7 +7971,7 @@ TEST_P(XdsSecurityTest, auto* transport_socket = cluster.mutable_transport_socket(); transport_socket->set_name("envoy.transport_sockets.tls"); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8010,7 +7994,7 @@ TEST_P( *validation_context->add_match_subject_alt_names() = server_san_exact_; transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8032,7 +8016,7 @@ TEST_P( ->set_instance_name(std::string("fake_plugin1")); transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8062,7 +8046,7 @@ TEST_P(XdsSecurityTest, RegexSanMatcherDoesNotAllowIgnoreCase) { *validation_context->add_match_subject_alt_names() = matcher; transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8082,7 +8066,7 @@ TEST_P(XdsSecurityTest, UnknownRootCertificateProvider) { ->set_instance_name("unknown"); transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8108,7 +8092,7 @@ TEST_P(XdsSecurityTest, UnknownIdentityCertificateProvider) { ->set_instance_name("fake_plugin1"); transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context); balancers_[0]->ads_service()->SetCdsResource(cluster); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->cds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8500,10 +8484,7 @@ TEST_P(XdsEnabledServerTest, BadLdsUpdateNoApiListenerNorAddress) { listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom( HttpConnectionManager()); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8527,10 +8508,7 @@ TEST_P(XdsEnabledServerTest, BadLdsUpdateBothApiListenerAndAddress) { HttpConnectionManager()); listener.mutable_api_listener(); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8551,10 +8529,7 @@ TEST_P(XdsEnabledServerTest, UnsupportedL4Filter) { backends_[0]->port()); listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(default_listener_ /* any proto object other than HttpConnectionManager */); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8583,10 +8558,7 @@ TEST_P(XdsEnabledServerTest, UnsupportedHttpFilter) { absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", backends_[0]->port())); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8616,10 +8588,7 @@ TEST_P(XdsEnabledServerTest, HttpFilterNotSupportedOnServer) { absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:", backends_[0]->port())); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8696,10 +8665,7 @@ TEST_P(XdsEnabledServerTest, UseOriginalDstNotSupported) { HttpConnectionManager()); listener.mutable_use_original_dst()->set_value(true); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8957,10 +8923,8 @@ TEST_P(XdsServerSecurityTest, UnknownTransportSocket) { auto* transport_socket = filter_chain->mutable_transport_socket(); transport_socket->set_name("unknown_transport_socket"); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -8992,10 +8956,8 @@ TEST_P( downstream_tls_context.mutable_require_client_certificate()->set_value(true); transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -9023,10 +8985,8 @@ TEST_P(XdsServerSecurityTest, DownstreamTlsContext downstream_tls_context; transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context); balancers_[0]->ads_service()->SetLdsResource(listener); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -9040,10 +9000,8 @@ TEST_P(XdsServerSecurityTest, UnknownIdentityCertificateProvider) { SetLdsUpdate("", "", "unknown", "", false); SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, true /* test_expects_failure */); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -9056,10 +9014,8 @@ TEST_P(XdsServerSecurityTest, UnknownRootCertificateProvider) { FakeCertificateProvider::CertDataMap fake1_cert_map = { {"", {root_cert_, identity_pair_}}}; SetLdsUpdate("unknown", "", "fake_plugin1", "", false); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state == - AdsServiceImpl::ResponseState::SENT); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->lds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -9832,12 +9788,8 @@ TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchNacked) { filter_chain->add_filters()->mutable_typed_config()->PackFrom( HttpConnectionManager()); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, ::testing::HasSubstr( @@ -9877,12 +9829,8 @@ TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnPrefixRangesNacked) { prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1"); prefix_range->mutable_prefix_len()->set_value(32); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; if (ipv6_only_) { EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, @@ -9922,12 +9870,8 @@ TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnTransportProtocolNacked) { filter_chain->mutable_filter_chain_match()->set_transport_protocol( "raw_buffer"); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, ::testing::HasSubstr("Duplicate matching rules detected when adding " @@ -9955,12 +9899,8 @@ TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnLocalSourceTypeNacked) { filter_chain->mutable_filter_chain_match()->set_source_type( FilterChainMatch::SAME_IP_OR_LOOPBACK); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, ::testing::HasSubstr("Duplicate matching rules detected when adding " @@ -9989,12 +9929,8 @@ TEST_P(XdsServerFilterChainMatchTest, filter_chain->mutable_filter_chain_match()->set_source_type( FilterChainMatch::EXTERNAL); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, ::testing::HasSubstr("Duplicate matching rules detected when adding " @@ -10035,12 +9971,8 @@ TEST_P(XdsServerFilterChainMatchTest, prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1"); prefix_range->mutable_prefix_len()->set_value(32); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; if (ipv6_only_) { EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, @@ -10078,12 +10010,8 @@ TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnSourcePortNacked) { HttpConnectionManager()); filter_chain->mutable_filter_chain_match()->add_source_ports(8080); balancers_[0]->ads_service()->SetLdsResource(listener); - auto initial_time = absl::Now(); - do { - CheckRpcSendFailure(); - } while (balancers_[0]->ads_service()->lds_response_state().state != - AdsServiceImpl::ResponseState::NACKED && - initial_time + absl::Seconds(60) > absl::Now()); + ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED)) + << "timed out waiting for NACK"; EXPECT_THAT( balancers_[0]->ads_service()->lds_response_state().error_message, ::testing::HasSubstr("Duplicate matching rules detected when adding " @@ -10101,7 +10029,7 @@ TEST_P(EdsTest, NacksSparsePriorityList) { {"locality0", CreateEndpointsForBackends(), kDefaultLocalityWeight, 1}, }); balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args)); - CheckRpcSendFailure(); + ASSERT_TRUE(WaitForEdsNack()) << "timed out waiting for NACK"; const auto response_state = balancers_[0]->ads_service()->eds_response_state(); EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED); @@ -11130,8 +11058,9 @@ TEST_P(ClientLoadReportingTest, Vanilla) { std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); - CheckRpcSendFailure(kNumFailuresPerAddress * num_backends_, - RpcOptions().set_server_fail(true)); + CheckRpcSendFailure(CheckRpcSendFailureOptions() + .set_times(kNumFailuresPerAddress * num_backends_) + .set_rpc_options(RpcOptions().set_server_fail(true))); // Check that each backend got the right number of requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress, @@ -11177,8 +11106,9 @@ TEST_P(ClientLoadReportingTest, SendAllClusters) { std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); - CheckRpcSendFailure(kNumFailuresPerAddress * num_backends_, - RpcOptions().set_server_fail(true)); + CheckRpcSendFailure(CheckRpcSendFailureOptions() + .set_times(kNumFailuresPerAddress * num_backends_) + .set_rpc_options(RpcOptions().set_server_fail(true))); // Check that each backend got the right number of requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress, @@ -11451,8 +11381,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) { // Config fault injection via different setup SetFilterConfig(http_fault); // Fire several RPCs, and expect all of them to be aborted. - CheckRpcSendFailure(5, RpcOptions().set_wait_for_ready(true), - StatusCode::ABORTED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_times(5) + .set_rpc_options(RpcOptions().set_wait_for_ready(true)) + .set_expected_error_code(StatusCode::ABORTED)); } // Without the listener config, the fault injection won't be enabled. @@ -12466,8 +12399,10 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) { TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) { int kTimeoutMillisecond = 1000; balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).xds_config(), ::testing::Contains(::testing::Property( @@ -12500,8 +12435,10 @@ TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) { routes2->mutable_route()->set_cluster(kClusterName2); SetRouteConfiguration(0, route_config); // Try to get the configs plumb through - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - StatusCode::DEADLINE_EXCEEDED); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).xds_config(), ::testing::Contains(::testing::Property( @@ -12528,8 +12465,10 @@ class CsdsShortAdsTimeoutTest : public ClientStatusDiscoveryServiceTest { TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) { int kTimeoutMillisecond = 1000000; // 1000s wait for the transient failure. balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName); - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - grpc::UNAVAILABLE); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(grpc::UNAVAILABLE)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).xds_config(), ::testing::Contains(::testing::Property( @@ -12548,8 +12487,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) { SetNextResolutionForLbChannelAllBalancers(); balancers_[0]->ads_service()->UnsetResource(kRdsTypeUrl, kDefaultRouteConfigurationName); - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - grpc::UNAVAILABLE); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(grpc::UNAVAILABLE)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT( csds_response.config(0).xds_config(), @@ -12566,8 +12507,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) { SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName); - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - grpc::UNAVAILABLE); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(grpc::UNAVAILABLE)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT(csds_response.config(0).xds_config(), ::testing::Contains(::testing::Property( @@ -12585,8 +12528,10 @@ TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) { SetNextResolutionForLbChannelAllBalancers(); balancers_[0]->ads_service()->UnsetResource(kEdsTypeUrl, kDefaultEdsServiceName); - CheckRpcSendFailure(1, RpcOptions().set_timeout_ms(kTimeoutMillisecond), - grpc::UNAVAILABLE); + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond)) + .set_expected_error_code(grpc::UNAVAILABLE)); auto csds_response = FetchCsdsResponse(); EXPECT_THAT( csds_response.config(0).xds_config(),