@ -46,15 +46,14 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, OutlierDetectionTest,
// test/cpp/end2end/outlier_detection_end2end_test.cc
// Tests SuccessRateEjectionAndUnejection:
// 1. Use ring hash policy that hashes using a
// header value to ensure rpcs go to all backends.
// 2. Cause a single error on 1
// backend and wait for 1 outlier detection interval to pass.
// 3. We should skip
// exactly 1 backend due to ejection and all the loads sticky to that backend
// should go to 1 other backend.
// 1. Use ring hash policy that hashes using a header value to ensure rpcs
// go to all backends.
// 2. Cause a single error on 1 backend and wait for 1 outlier detection
// interval to pass.
// 3. We should skip exactly 1 backend due to ejection and all the loads
// sticky to that backend should go to 1 other backend.
// 4. Let the ejection period pass and verify we can go back to both backends
// after the uneject.
// after the uneject.
TEST_P ( OutlierDetectionTest , SuccessRateEjectionAndUnejection ) {
ScopedExperimentalEnvVar env_var (
" GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION " ) ;
@ -500,15 +499,14 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) {
}
// Tests FailurePercentageEjectionAndUnejection:
// 1. Use ring hash policy that hashes using a
// header value to ensure rpcs go to all backends.
// 2. Cause a single error on 1
// backend and wait for 1 outlier detection interval to pass.
// 3. We should skip
// exactly 1 backend due to ejection and all the loads sticky to that backend
// should go to 1 other backend.
// 1. Use ring hash policy that hashes using a header value to ensure RPCs
// go to all backends.
// 2. Cause a single error on 1 backend and wait for 1 outlier detection
// interval to pass.
// 3. We should skip exactly 1 backend due to ejection and all the loads
// sticky to that backend should go to 1 other backend.
// 4. Let the ejection period pass and verify that traffic will again go both
// backends as we have unejected the backend.
// backends as we have unejected the backend.
TEST_P ( OutlierDetectionTest , FailurePercentageEjectionAndUnejection ) {
ScopedExperimentalEnvVar env_var (
" GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION " ) ;
@ -1237,6 +1235,86 @@ TEST_P(OutlierDetectionTest,
EXPECT_EQ ( 100 , backends_ [ 3 ] - > backend_service ( ) - > request_count ( ) ) ;
}
// Tests that we uneject any ejected addresses when the OD policy is
// disabled.
TEST_P ( OutlierDetectionTest , DisableOutlierDetectionWhileAddressesAreEjected ) {
ScopedExperimentalEnvVar env_var (
" GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION " ) ;
CreateAndStartBackends ( 2 ) ;
auto cluster = default_cluster_ ;
cluster . set_lb_policy ( Cluster : : RING_HASH ) ;
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto * interval = cluster . mutable_outlier_detection ( ) - > mutable_interval ( ) ;
interval - > set_nanos ( 100000000 * grpc_test_slowdown_factor ( ) ) ;
auto * base_time =
cluster . mutable_outlier_detection ( ) - > mutable_base_ejection_time ( ) ;
base_time - > set_seconds ( 1 * grpc_test_slowdown_factor ( ) ) ;
cluster . mutable_outlier_detection ( )
- > mutable_failure_percentage_threshold ( )
- > set_value ( 0 ) ;
cluster . mutable_outlier_detection ( )
- > mutable_enforcing_failure_percentage ( )
- > set_value ( 100 ) ;
cluster . mutable_outlier_detection ( )
- > mutable_failure_percentage_minimum_hosts ( )
- > set_value ( 1 ) ;
cluster . mutable_outlier_detection ( )
- > mutable_failure_percentage_request_volume ( )
- > set_value ( 1 ) ;
balancer_ - > ads_service ( ) - > SetCdsResource ( cluster ) ;
auto new_route_config = default_route_config_ ;
auto * route = new_route_config . mutable_virtual_hosts ( 0 ) - > mutable_routes ( 0 ) ;
auto * hash_policy = route - > mutable_route ( ) - > add_hash_policy ( ) ;
hash_policy - > mutable_header ( ) - > set_header_name ( " address_hash " ) ;
SetListenerAndRouteConfiguration ( balancer_ . get ( ) , default_listener_ ,
new_route_config ) ;
EdsResourceArgs args ( { { " locality0 " , CreateEndpointsForBackends ( ) } } ) ;
balancer_ - > ads_service ( ) - > SetEdsResource ( BuildEdsResource ( args ) ) ;
// Note each type of RPC will contains a header value that will always be
// hashed to a specific backend as the header value matches the value used
// to create the entry in the ring.
std : : vector < std : : pair < std : : string , std : : string > > metadata = {
{ " address_hash " , CreateMetadataValueThatHashesToBackend ( 0 ) } } ;
std : : vector < std : : pair < std : : string , std : : string > > metadata1 = {
{ " address_hash " , CreateMetadataValueThatHashesToBackend ( 1 ) } } ;
const auto rpc_options = RpcOptions ( ) . set_metadata ( metadata ) ;
const auto rpc_options1 = RpcOptions ( ) . set_metadata ( std : : move ( metadata1 ) ) ;
WaitForBackend ( DEBUG_LOCATION , 0 , /*check_status=*/ nullptr ,
WaitForBackendOptions ( ) , rpc_options ) ;
WaitForBackend ( DEBUG_LOCATION , 1 , /*check_status=*/ nullptr ,
WaitForBackendOptions ( ) , rpc_options1 ) ;
// Cause an error and wait for 1 outlier detection interval to pass to cause
// the backend to be ejected.
CheckRpcSendFailure (
DEBUG_LOCATION , StatusCode : : CANCELLED , " " ,
RpcOptions ( ) . set_metadata ( metadata ) . set_server_expected_error (
StatusCode : : CANCELLED ) ) ;
gpr_sleep_until ( grpc_timeout_milliseconds_to_deadline ( 100 ) ) ;
ResetBackendCounters ( ) ;
// 1 backend is ejected all traffic going to the ejected backend should now
// all be going to the other backend.
// failure percentage enforcement_percentage of 100% is honored as this test
// will consistently reject 1 backend.
CheckRpcSendOk ( DEBUG_LOCATION , 1 , rpc_options ) ;
EXPECT_EQ ( 1 , backends_ [ 1 ] - > backend_service ( ) - > request_count ( ) ) ;
// Send an update that disables outlier detection.
cluster . clear_outlier_detection ( ) ;
balancer_ - > ads_service ( ) - > SetCdsResource ( cluster ) ;
// Wait for the backend to start being used again.
WaitForBackend (
DEBUG_LOCATION , 0 ,
[ ] ( const RpcResult & result ) {
EXPECT_EQ ( result . status . error_code ( ) , StatusCode : : CANCELLED )
< < " Error: " < < result . status . error_message ( ) ;
} ,
WaitForBackendOptions ( ) ,
RpcOptions ( )
. set_metadata ( std : : move ( metadata ) )
. set_server_expected_error ( StatusCode : : CANCELLED ) ) ;
}
} // namespace
} // namespace testing
} // namespace grpc
Mark D. Roth
3 years ago
committed by
GitHub