Add outlier detection xDS interop test using k8s interop framework (#30250)

* Add xDS interop test for outlier detection

This implements the test described in #29623, and plumbing for setting the
outlierDetection field in the backend service config. The changes in this PR
are very similar to #29688.

* Fix use of configure method

* Correct copy/paste error

* Fix metadata configuration syntax

* Increase QPS, use just one method

* Format code

* Apply suggestions from code review

Co-authored-by: Sergii Tkachenko <hi@sergii.org>

* Address review comments

* Only Java implements the required server features

* Automated change: Fix sanity tests

* Address review comments

* Use double quotes for docstring

Co-authored-by: Sergii Tkachenko <hi@sergii.org>

Co-authored-by: Sergii Tkachenko <hi@sergii.org>
Co-authored-by: murgatroid99 <murgatroid99@users.noreply.github.com>

tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py

@@ -121,7 +121,8 @@ class ComputeV1(gcp.api.GcpProjectApiResource):  # pylint: disable=too-many-publ
             affinity_header: Optional[str] = None,
             protocol: Optional[BackendServiceProtocol] = None,
             subset_size: Optional[int] = None,
-            locality_lb_policies: Optional[List[dict]] = None) -> 'GcpResource':
+            locality_lb_policies: Optional[List[dict]] = None,
+            outlier_detection: Optional[dict] = None) -> 'GcpResource':
         if not isinstance(protocol, self.BackendServiceProtocol):
             raise TypeError(f'Unexpected Backend Service protocol: {protocol}')
         body = {
@@ -145,6 +146,8 @@ class ComputeV1(gcp.api.GcpProjectApiResource):  # pylint: disable=too-many-publ
             }
         if locality_lb_policies:
             body['localityLbPolicies'] = locality_lb_policies
+        if outlier_detection:
+            body['outlierDetection'] = outlier_detection
         return self._insert_resource(self.api.backendServices(), body)
 
     def get_backend_service_traffic_director(self, name: str) -> 'GcpResource':

tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py

@@ -196,7 +196,8 @@ class TrafficDirectorManager:  # pylint: disable=too-many-public-methods
             protocol: Optional[BackendServiceProtocol] = _BackendGRPC,
             subset_size: Optional[int] = None,
             affinity_header: Optional[str] = None,
-            locality_lb_policies: Optional[List[dict]] = None):
+            locality_lb_policies: Optional[List[dict]] = None,
+            outlier_detection: Optional[dict] = None):
         if protocol is None:
             protocol = _BackendGRPC
 
@@ -208,7 +209,8 @@ class TrafficDirectorManager:  # pylint: disable=too-many-public-methods
             protocol=protocol,
             subset_size=subset_size,
             affinity_header=affinity_header,
-            locality_lb_policies=locality_lb_policies)
+            locality_lb_policies=locality_lb_policies,
+            outlier_detection=outlier_detection)
         self.backend_service = resource
         self.backend_service_protocol = protocol
 

tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml

@@ -44,6 +44,8 @@ spec:
             value: "true"
           - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG
             value: "true"
+          - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION
+            value: "true"
         volumeMounts:
           - mountPath: /tmp/grpc-xds/
             name: grpc-td-conf

tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py

@@ -0,0 +1,115 @@
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import List
+
+from absl import flags
+from absl.testing import absltest
+
+from framework import xds_k8s_testcase
+from framework import xds_url_map_testcase
+from framework.helpers import skips
+
+logger = logging.getLogger(__name__)
+flags.adopt_module_key_flags(xds_k8s_testcase)
+
+# Type aliases
+RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
+RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
+_XdsTestServer = xds_k8s_testcase.XdsTestServer
+_XdsTestClient = xds_k8s_testcase.XdsTestClient
+_Lang = skips.Lang
+
+# Testing consts
+_QPS = 100
+_REPLICA_COUNT = 5
+
+
+class OutlierDetectionTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
+    """
+    Implementation of https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#outlier_detection
+
+    This test verifies that the client applies the outlier detection
+    configuration and temporarily drops traffic to a server that fails
+    requests.
+    """
+
+    @staticmethod
+    def is_supported(config: skips.TestConfig) -> bool:
+        if config.server_lang != _Lang.JAVA:
+            return False
+        if config.client_lang in _Lang.CPP | _Lang.PYTHON:
+            return config.version_gte('v1.48.x')
+        if config.client_lang == _Lang.NODE:
+            return config.version_gte('v1.6.x')
+        return False
+
+    def test_outlier_detection(self) -> None:
+
+        with self.subTest('00_create_health_check'):
+            self.td.create_health_check()
+
+        with self.subTest('01_create_backend_service'):
+            self.td.create_backend_service(
+                outlier_detection={
+                    'interval': {
+                        'seconds': 2,
+                        'nanos': 0
+                    },
+                    'successRateRequestVolume': 20
+                })
+
+        with self.subTest('02_create_url_map'):
+            self.td.create_url_map(self.server_xds_host, self.server_xds_port)
+
+        with self.subTest('03_create_target_proxy'):
+            self.td.create_target_proxy()
+
+        with self.subTest('04_create_forwarding_rule'):
+            self.td.create_forwarding_rule(self.server_xds_port)
+
+        test_servers: List[_XdsTestServer]
+        with self.subTest('05_start_test_servers'):
+            test_servers = self.startTestServers(replica_count=_REPLICA_COUNT)
+
+        with self.subTest('06_add_server_backends_to_backend_services'):
+            self.setupServerBackends()
+
+        test_client: _XdsTestClient
+        with self.subTest('07_start_test_client'):
+            test_client = self.startTestClient(test_servers[0], qps=_QPS)
+
+        with self.subTest('08_test_client_xds_config_exists'):
+            self.assertXdsConfigExists(test_client)
+
+        with self.subTest('09_test_servers_received_rpcs_from_test_client'):
+            self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers)
+
+        rpc_types = (RpcTypeUnaryCall,)
+        with self.subTest('10_chosen_server_removed_by_outlier_detection'):
+            test_client.update_config.configure(
+                rpc_types=rpc_types,
+                metadata=(
+                    (RpcTypeUnaryCall, 'rpc-behavior',
+                     f'hostname={test_servers[0].pod_name} error-code-2'),))
+            self.assertRpcsEventuallyGoToGivenServers(test_client,
+                                                      test_servers[1:])
+
+        with self.subTest('11_ejected_server_returned_after_failures_stopped'):
+            test_client.update_config.configure(rpc_types=rpc_types)
+            self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers)
+
+
+if __name__ == '__main__':
+    absltest.main(failfast=True)