Allow more advanced test case support semantic (#27373)

* Allow more advanced test case support semantic

* Fix comments and docs

* Correct the supported Golang version in retry test

* Make isort happy

tools/internal_ci/linux/grpc_xds_url_map.sh

@@ -91,6 +91,7 @@ run_test() {
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
     --kube_context="${KUBE_CONTEXT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
+    --testing_version=$(echo "$KOKORO_JOB_NAME" | sed -E 's|^grpc/core/([^/]+)/.*|\1|') \
     --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
     --flagfile="config/url-map.cfg"
   set +x

tools/internal_ci/linux/grpc_xds_url_map_python.sh

@@ -101,6 +101,7 @@ run_test() {
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
     --kube_context="${KUBE_CONTEXT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
+    --testing_version=$(echo "$KOKORO_JOB_NAME" | sed -E 's|^grpc/core/([^/]+)/.*|\1|') \
     --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
     --flagfile="config/url-map.cfg"
   set +x

tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py

@@ -109,6 +109,12 @@ CLIENT_PORT = flags.DEFINE_integer(
         "XdsStats, XdsUpdateClientConfigure, and ProtoReflection (optional).\n"
         "Doesn't have to be within --firewall_allowed_ports."))
 
+# Testing metadata
+TESTING_VERSION = flags.DEFINE_string(
+    "testing_version",
+    default="master",
+    help="The testing gRPC version branch name. Like master, v1.41.x, v1.37.x")
+
 flags.adopt_module_key_flags(highlighter)
 
 flags.mark_flags_as_required([

tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py

@@ -29,6 +29,7 @@ from absl import logging
 from absl.testing import absltest
 from google.protobuf import json_format
 import grpc
+import packaging
 
 from framework import xds_k8s_testcase
 from framework import xds_url_map_test_resources
@@ -62,10 +63,6 @@ JsonType = Any
 RpcTypeUnaryCall = 'UNARY_CALL'
 RpcTypeEmptyCall = 'EMPTY_CALL'
 
-# All client languages
-_CLIENT_LANGUAGES = ('cpp', 'java', 'go', 'python')
-_SERVER_LANGUAGES = _CLIENT_LANGUAGES
-
 
 def _split_camel(s: str, delimiter: str = '-') -> str:
     """Turn camel case name to snake-case-like name."""
@@ -188,6 +185,27 @@ class ExpectedResult:
     ratio: float = 1
 
 
+@dataclass
+class TestConfig:
+    """Describes the config for the test suite."""
+    client_lang: str
+    server_lang: str
+    version: str
+
+    def version_ge(self, another: str) -> bool:
+        """Returns a bool for whether the version is >= another one.
+
+        A version is greater than or equal to another version means its version
+        number is greater than or equal to another version's number. Version
+        "master" is always considered latest. E.g., master >= v1.41.x >= v1.40.x
+        >= v1.9.x.
+        """
+        if self.version == 'master':
+            return True
+        return packaging.version.parse(
+            self.version) >= packaging.version.parse(another)
+
+
 class _MetaXdsUrlMapTestCase(type):
     """Tracking test case subclasses."""
 
@@ -233,22 +251,13 @@ class XdsUrlMapTestCase(absltest.TestCase, metaclass=_MetaXdsUrlMapTestCase):
     """
 
     @staticmethod
-    def supported_clients() -> Tuple[str]:
-        """Declare supported languages of clients.
+    def is_supported(config: TestConfig) -> bool:
+        """Allow the test case to decide whether it supports the given config.
 
         Returns:
-          A tuple of strings contains the supported languages for this test.
+          A bool indicates if the given config is supported.
         """
-        return _CLIENT_LANGUAGES
-
-    @staticmethod
-    def supported_servers() -> Tuple[str]:
-        """Declare supported languages of servers.
-
-        Returns:
-          A tuple of strings contains the supported languages for this test.
-        """
-        return _SERVER_LANGUAGES
+        return True
 
     @staticmethod
     def client_init_config(rpc: str, metadata: str) -> Tuple[str, str]:
@@ -336,15 +345,12 @@ class XdsUrlMapTestCase(absltest.TestCase, metaclass=_MetaXdsUrlMapTestCase):
         # cannot be used in the built-in test-skipping decorators. See the
         # official FAQs:
         # https://abseil.io/docs/python/guides/flags#faqs
-        client_lang = _get_lang(GcpResourceManager().client_image)
-        server_lang = _get_lang(GcpResourceManager().server_image)
-        if client_lang not in cls.supported_clients():
-            cls.skip_reason = (f'Unsupported client language {client_lang} '
-                               f'not in {cls.supported_clients()}')
-            return
-        elif server_lang not in cls.supported_servers():
-            cls.skip_reason = (f'Unsupported server language {server_lang} '
-                               f'not in {cls.supported_servers()}')
+        test_config = TestConfig(
+            client_lang=_get_lang(GcpResourceManager().client_image),
+            server_lang=_get_lang(GcpResourceManager().server_image),
+            version=GcpResourceManager().testing_version)
+        if not cls.is_supported(test_config):
+            cls.skip_reason = f'Unsupported test config: {test_config}'
             return
         else:
             cls.skip_reason = None

tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py

@@ -31,6 +31,7 @@ GcpResourceManager = xds_url_map_testcase.GcpResourceManager
 DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
 RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
 RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
+TestConfig = xds_url_map_testcase.TestConfig
 XdsTestClient = client_app.XdsTestClient
 
 logger = logging.getLogger(__name__)
@@ -56,8 +57,12 @@ _ChannelzChannelState = grpc_channelz.ChannelState
 class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
-    def supported_clients() -> Tuple[str]:
-        return 'cpp', 'java', 'go'
+    def is_supported(config: TestConfig) -> bool:
+        if config.client_lang in ['cpp', 'java']:
+            return config.version_ge('v1.40.x')
+        if config.client_lang in ['go']:
+            return config.version_ge('v1.41.x')
+        return False
 
     @staticmethod
     def client_init_config(rpc: str, metadata: str):
@@ -121,8 +126,12 @@ class TestHeaderBasedAffinityMultipleHeaders(
         xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
-    def supported_clients() -> Tuple[str]:
-        return 'cpp', 'java', 'go'
+    def is_supported(config: TestConfig) -> bool:
+        if config.client_lang in ['cpp', 'java']:
+            return config.version_ge('v1.40.x')
+        if config.client_lang in ['go']:
+            return config.version_ge('v1.41.x')
+        return False
 
     @staticmethod
     def client_init_config(rpc: str, metadata: str):

tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py

@@ -31,6 +31,7 @@ DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
 RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
 XdsTestClient = client_app.XdsTestClient
 ExpectedResult = xds_url_map_testcase.ExpectedResult
+TestConfig = xds_url_map_testcase.TestConfig
 
 logger = logging.getLogger(__name__)
 flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -66,8 +67,12 @@ def _build_retry_route_rule(retryConditions, num_retries):
 class TestRetryUpTo3AttemptsAndFail(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
-    def supported_clients() -> Tuple[str]:
-        return 'cpp', 'java', 'go'
+    def is_supported(config: TestConfig) -> bool:
+        if config.client_lang in ['cpp', 'java']:
+            return config.version_ge('v1.40.x')
+        elif config.client_lang == 'go':
+            return config.version_ge('v1.41.x')
+        return False
 
     @staticmethod
     def url_map_change(
@@ -105,8 +110,12 @@ class TestRetryUpTo3AttemptsAndFail(xds_url_map_testcase.XdsUrlMapTestCase):
 class TestRetryUpTo4AttemptsAndSucceed(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
-    def supported_clients() -> Tuple[str]:
-        return 'cpp', 'java', 'go'
+    def is_supported(config: TestConfig) -> bool:
+        if config.client_lang in ['cpp', 'java']:
+            return config.version_ge('v1.40.x')
+        elif config.client_lang == 'go':
+            return config.version_ge('v1.41.x')
+        return False
 
     @staticmethod
     def url_map_change(

tools/run_tests/xds_k8s_test_driver/tests/url_map/timeout_test.py

@@ -34,6 +34,7 @@ RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
 ExpectedResult = xds_url_map_testcase.ExpectedResult
 XdsTestClient = client_app.XdsTestClient
 XdsUrlMapTestCase = xds_url_map_testcase.XdsUrlMapTestCase
+TestConfig = xds_url_map_testcase.TestConfig
 
 logger = logging.getLogger(__name__)
 flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -81,10 +82,10 @@ class _BaseXdsTimeOutTestCase(XdsUrlMapTestCase):
 class TestTimeoutInRouteRule(_BaseXdsTimeOutTestCase):
 
     @staticmethod
-    def supported_servers() -> Tuple[str]:
+    def is_supported(config: TestConfig) -> bool:
         # TODO(lidiz) either add support for rpc-behavior to other languages, or we
         # should always use Java server as backend.
-        return 'java',
+        return config.server_lang == 'java'
 
     def rpc_distribution_validate(self, test_client: XdsTestClient):
         rpc_distribution = self.configure_and_send(
@@ -112,8 +113,8 @@ class TestTimeoutInRouteRule(_BaseXdsTimeOutTestCase):
 class TestTimeoutInApplication(_BaseXdsTimeOutTestCase):
 
     @staticmethod
-    def supported_servers() -> Tuple[str]:
-        return 'java',
+    def is_supported(config: TestConfig) -> bool:
+        return config.server_lang == 'java'
 
     def rpc_distribution_validate(self, test_client: XdsTestClient):
         rpc_distribution = self.configure_and_send(