diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py index faffeb477e2..0387d0b1244 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py @@ -121,7 +121,8 @@ class ComputeV1(gcp.api.GcpProjectApiResource): # pylint: disable=too-many-publ affinity_header: Optional[str] = None, protocol: Optional[BackendServiceProtocol] = None, subset_size: Optional[int] = None, - locality_lb_policies: Optional[List[dict]] = None) -> 'GcpResource': + locality_lb_policies: Optional[List[dict]] = None, + outlier_detection: Optional[dict] = None) -> 'GcpResource': if not isinstance(protocol, self.BackendServiceProtocol): raise TypeError(f'Unexpected Backend Service protocol: {protocol}') body = { @@ -145,6 +146,8 @@ class ComputeV1(gcp.api.GcpProjectApiResource): # pylint: disable=too-many-publ } if locality_lb_policies: body['localityLbPolicies'] = locality_lb_policies + if outlier_detection: + body['outlierDetection'] = outlier_detection return self._insert_resource(self.api.backendServices(), body) def get_backend_service_traffic_director(self, name: str) -> 'GcpResource': diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py index 431f8f58aad..fbb791aa15b 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py @@ -196,7 +196,8 @@ class TrafficDirectorManager: # pylint: disable=too-many-public-methods protocol: Optional[BackendServiceProtocol] = _BackendGRPC, subset_size: Optional[int] = None, affinity_header: Optional[str] = None, - locality_lb_policies: Optional[List[dict]] = None): + locality_lb_policies: Optional[List[dict]] = None, + outlier_detection: Optional[dict] = None): if protocol is None: protocol = _BackendGRPC @@ -208,7 +209,8 @@ class TrafficDirectorManager: # pylint: disable=too-many-public-methods protocol=protocol, subset_size=subset_size, affinity_header=affinity_header, - locality_lb_policies=locality_lb_policies) + locality_lb_policies=locality_lb_policies, + outlier_detection=outlier_detection) self.backend_service = resource self.backend_service_protocol = protocol diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml index 3246f914660..5820803990a 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml @@ -44,6 +44,8 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG value: "true" + - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION + value: "true" volumeMounts: - mountPath: /tmp/grpc-xds/ name: grpc-td-conf diff --git a/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py b/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py new file mode 100644 index 00000000000..ce26e01d28e --- /dev/null +++ b/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py @@ -0,0 +1,115 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import List + +from absl import flags +from absl.testing import absltest + +from framework import xds_k8s_testcase +from framework import xds_url_map_testcase +from framework.helpers import skips + +logger = logging.getLogger(__name__) +flags.adopt_module_key_flags(xds_k8s_testcase) + +# Type aliases +RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall +RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall +_XdsTestServer = xds_k8s_testcase.XdsTestServer +_XdsTestClient = xds_k8s_testcase.XdsTestClient +_Lang = skips.Lang + +# Testing consts +_QPS = 100 +_REPLICA_COUNT = 5 + + +class OutlierDetectionTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): + """ + Implementation of https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#outlier_detection + + This test verifies that the client applies the outlier detection + configuration and temporarily drops traffic to a server that fails + requests. + """ + + @staticmethod + def is_supported(config: skips.TestConfig) -> bool: + if config.server_lang != _Lang.JAVA: + return False + if config.client_lang in _Lang.CPP | _Lang.PYTHON: + return config.version_gte('v1.48.x') + if config.client_lang == _Lang.NODE: + return config.version_gte('v1.6.x') + return False + + def test_outlier_detection(self) -> None: + + with self.subTest('00_create_health_check'): + self.td.create_health_check() + + with self.subTest('01_create_backend_service'): + self.td.create_backend_service( + outlier_detection={ + 'interval': { + 'seconds': 2, + 'nanos': 0 + }, + 'successRateRequestVolume': 20 + }) + + with self.subTest('02_create_url_map'): + self.td.create_url_map(self.server_xds_host, self.server_xds_port) + + with self.subTest('03_create_target_proxy'): + self.td.create_target_proxy() + + with self.subTest('04_create_forwarding_rule'): + self.td.create_forwarding_rule(self.server_xds_port) + + test_servers: List[_XdsTestServer] + with self.subTest('05_start_test_servers'): + test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) + + with self.subTest('06_add_server_backends_to_backend_services'): + self.setupServerBackends() + + test_client: _XdsTestClient + with self.subTest('07_start_test_client'): + test_client = self.startTestClient(test_servers[0], qps=_QPS) + + with self.subTest('08_test_client_xds_config_exists'): + self.assertXdsConfigExists(test_client) + + with self.subTest('09_test_servers_received_rpcs_from_test_client'): + self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers) + + rpc_types = (RpcTypeUnaryCall,) + with self.subTest('10_chosen_server_removed_by_outlier_detection'): + test_client.update_config.configure( + rpc_types=rpc_types, + metadata=( + (RpcTypeUnaryCall, 'rpc-behavior', + f'hostname={test_servers[0].pod_name} error-code-2'),)) + self.assertRpcsEventuallyGoToGivenServers(test_client, + test_servers[1:]) + + with self.subTest('11_ejected_server_returned_after_failures_stopped'): + test_client.update_config.configure(rpc_types=rpc_types) + self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers) + + +if __name__ == '__main__': + absltest.main(failfast=True)