xds-k8s: Fix the issue with Java PSM security tests skipped (#29925)

1. Fixes the issue with Java PSM security tests accidentally skipped because Java was missing from the list of languages, ref https://github.com/grpc/grpc/pull/28978
2. Invert the logic of `is_supported` methods, making them normally open
3. Make languages an `enum.Flag` to avoid accidental typos when listing the languages
4. Rename `XdsKubernetesTestCase.isSupported` to `XdsKubernetesTestCase.is_supported` to be consistent with `XdsUrlMapTestCase.is_supported`
5. Add extra logging

tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 """The classes and predicates to assist validate test config for test cases."""
 from dataclasses import dataclass
+import enum
+import logging
 import re
-from typing import Callable
+from typing import Callable, Optional
 import unittest
 
 from packaging import version as pkg_version
@@ -22,35 +24,75 @@ from packaging import version as pkg_version
 from framework import xds_flags
 from framework import xds_k8s_flags
 
+logger = logging.getLogger(__name__)
 
-def _get_lang(image_name: str) -> str:
-    return re.search(r'/(\w+)-(client|server):', image_name).group(1)
 
+class Lang(enum.Flag):
+    UNKNOWN = enum.auto()
+    CPP = enum.auto()
+    GO = enum.auto()
+    JAVA = enum.auto()
+    PYTHON = enum.auto()
+    NODE = enum.auto()
 
-def _parse_version(s: str) -> pkg_version.Version:
-    if s.endswith(".x"):
-        s = s[:-2]
-    return pkg_version.Version(s)
+    def __str__(self):
+        return str(self.name).lower()
+
+    @classmethod
+    def from_string(cls, lang: str):
+        try:
+            return cls[lang.upper()]
+        except KeyError:
+            return cls.UNKNOWN
 
 
 @dataclass
 class TestConfig:
     """Describes the config for the test suite."""
-    client_lang: str
-    server_lang: str
-    version: str
+    client_lang: Lang
+    server_lang: Lang
+    version: Optional[str]
 
-    def version_ge(self, another: str) -> bool:
+    def version_gte(self, another: str) -> bool:
         """Returns a bool for whether the version is >= another one.
 
         A version is greater than or equal to another version means its version
         number is greater than or equal to another version's number. Version
-        "master" is always considered latest. E.g., master >= v1.41.x >= v1.40.x
-        >= v1.9.x.
+        "master" is always considered latest.
+        E.g., master >= v1.41.x >= v1.40.x >= v1.9.x.
+
+        Unspecified version is treated as 'master', but isn't explicitly set.
         """
-        if self.version == 'master':
+        if self.version == 'master' or self.version is None:
             return True
-        return _parse_version(self.version) >= _parse_version(another)
+        return self._parse_version(self.version) >= self._parse_version(another)
+
+    def version_lt(self, another: str) -> bool:
+        """Returns a bool for whether the version is < another one.
+
+        Version "master" is always considered latest.
+        E.g., v1.9.x < v1.40.x < v1.41.x < master.
+
+        Unspecified version is treated as 'master', but isn't explicitly set.
+        """
+        if self.version == 'master' or self.version is None:
+            return False
+        return self._parse_version(self.version) < self._parse_version(another)
+
+    def __str__(self):
+        return (f"TestConfig(client_lang='{self.client_lang}', "
+                f"server_lang='{self.server_lang}', version={self.version!r})")
+
+    @staticmethod
+    def _parse_version(s: str) -> pkg_version.Version:
+        if s.endswith(".x"):
+            s = s[:-2]
+        return pkg_version.Version(s)
+
+
+def _get_lang(image_name: str) -> Lang:
+    return Lang.from_string(
+        re.search(r'/(\w+)-(client|server):', image_name).group(1))
 
 
 def evaluate_test_config(check: Callable[[TestConfig], bool]) -> None:
@@ -64,4 +106,7 @@ def evaluate_test_config(check: Callable[[TestConfig], bool]) -> None:
         server_lang=_get_lang(xds_k8s_flags.SERVER_IMAGE.value),
         version=xds_flags.TESTING_VERSION.value)
     if not check(test_config):
+        logger.info('Skipping %s', test_config)
         raise unittest.SkipTest(f'Unsupported test config: {test_config}')
+
+    logger.info('Detected language and version: %s', test_config)

tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py

@@ -116,7 +116,7 @@ CLIENT_PORT = flags.DEFINE_integer(
 # Testing metadata
 TESTING_VERSION = flags.DEFINE_string(
     "testing_version",
-    default="master",
+    default=None,
     help="The testing gRPC version branch name. Like master, v1.41.x, v1.37.x")
 
 FORCE_CLEANUP = flags.DEFINE_bool(

tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py

@@ -85,7 +85,7 @@ class XdsKubernetesBaseTestCase(absltest.TestCase):
     td: TrafficDirectorManager
 
     @staticmethod
-    def isSupported(config: skips.TestConfig) -> bool:
+    def is_supported(config: skips.TestConfig) -> bool:
         """Overridden by the test class to decide if the config is supported.
 
         Returns:
@@ -104,7 +104,7 @@ class XdsKubernetesBaseTestCase(absltest.TestCase):
 
         # Raises unittest.SkipTest if given client/server/version does not
         # support current test case.
-        skips.evaluate_test_config(cls.isSupported)
+        skips.evaluate_test_config(cls.is_supported)
 
         # GCP
         cls.project: str = xds_flags.PROJECT.value

tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py

@@ -31,6 +31,7 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
 _XdsTestServer = xds_k8s_testcase.XdsTestServer
 _XdsTestClient = xds_k8s_testcase.XdsTestClient
 _ChannelzChannelState = grpc_channelz.ChannelState
+_Lang = skips.Lang
 
 # Testing consts
 _TEST_AFFINITY_METADATA_KEY = 'xds_md'
@@ -43,10 +44,12 @@ _RPC_COUNT = 100
 class AffinityTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
 
     @staticmethod
-    def isSupported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'java', 'python', 'go']:
-            return config.version_ge('v1.40.x')
-        return False
+    def is_supported(config: skips.TestConfig) -> bool:
+        if config.client_lang in (_Lang.CPP | _Lang.GO | _Lang.JAVA |
+                                  _Lang.PYTHON):
+            # Versions prior to v1.40.x don't support Affinity.
+            return not config.version_lt('v1.40.x')
+        return True
 
     def test_affinity(self) -> None:  # pylint: disable=too-many-statements
 

tools/run_tests/xds_k8s_test_driver/tests/authz_test.py

@@ -29,6 +29,7 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
 _XdsTestServer = xds_k8s_testcase.XdsTestServer
 _XdsTestClient = xds_k8s_testcase.XdsTestClient
 _SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode
+_Lang = skips.Lang
 
 # The client generates QPS even when it is still loading information from xDS.
 # Once it finally connects there will be an outpouring of the bufferred RPCs and
@@ -47,12 +48,14 @@ class AuthzTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
     }
 
     @staticmethod
-    def isSupported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'python']:
-            return config.version_ge('v1.44.x')
-        elif config.client_lang in ['java', 'go']:
-            return config.version_ge('v1.42.x')
-        return False
+    def is_supported(config: skips.TestConfig) -> bool:
+        # Per "Authorization (RBAC)" in
+        # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+        if config.client_lang in _Lang.CPP | _Lang.PYTHON:
+            return not config.version_lt('v1.44.x')
+        elif config.client_lang in _Lang.GO | _Lang.JAVA:
+            return not config.version_lt('v1.42.x')
+        return True
 
     def setUp(self):
         super().setUp()

tools/run_tests/xds_k8s_test_driver/tests/security_test.py

@@ -27,15 +27,19 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
 _XdsTestServer = xds_k8s_testcase.XdsTestServer
 _XdsTestClient = xds_k8s_testcase.XdsTestClient
 _SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode
+_Lang = skips.Lang
 
 
 class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
 
     @staticmethod
-    def isSupported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'python', 'go']:
-            return config.version_ge('v1.41.x')
-        return False
+    def is_supported(config: skips.TestConfig) -> bool:
+        if config.client_lang in (_Lang.CPP | _Lang.GO | _Lang.JAVA |
+                                  _Lang.PYTHON):
+            # Versions prior to v1.41.x don't support PSM Security.
+            # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+            return not config.version_lt('v1.41.x')
+        return True
 
     def test_mtls(self):
         """mTLS test.

tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py

@@ -38,11 +38,11 @@ _NUM_CLIENTS = 3
 class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
 
     @staticmethod
-    def isSupported(config: skips.TestConfig) -> bool:
+    def is_supported(config: skips.TestConfig) -> bool:
         # Subsetting is an experimental feature where most work is done on the
         # server-side. We limit it to only run on master branch to save
         # resources.
-        return config.version_ge('master')
+        return config.version_gte('master')
 
     def test_subsetting_basic(self) -> None:
         with self.subTest('00_create_health_check'):

tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py

@@ -31,6 +31,7 @@ DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
 RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
 RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
 XdsTestClient = client_app.XdsTestClient
+_Lang = skips.Lang
 
 logger = logging.getLogger(__name__)
 flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -52,15 +53,25 @@ _TEST_METADATA = (
 _ChannelzChannelState = grpc_channelz.ChannelState
 
 
+def _is_supported(config: skips.TestConfig) -> bool:
+    # Per "Ring hash" in
+    # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+    if config.client_lang in _Lang.CPP | _Lang.JAVA:
+        return not config.version_lt('v1.40.x')
+    elif config.client_lang == _Lang.GO:
+        return not config.version_lt('v1.41.x')
+    elif config.client_lang == _Lang.PYTHON:
+        # TODO(https://github.com/grpc/grpc/issues/27430): supported after
+        #      the issue is fixed.
+        return False
+    return True
+
+
 class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
     def is_supported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'java']:
-            return config.version_ge('v1.40.x')
-        if config.client_lang in ['go']:
-            return config.version_ge('v1.41.x')
-        return False
+        return _is_supported(config)
 
     @staticmethod
     def client_init_config(rpc: str, metadata: str):
@@ -125,11 +136,7 @@ class TestHeaderBasedAffinityMultipleHeaders(
 
     @staticmethod
     def is_supported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'java']:
-            return config.version_ge('v1.40.x')
-        if config.client_lang in ['go']:
-            return config.version_ge('v1.41.x')
-        return False
+        return _is_supported(config)
 
     @staticmethod
     def client_init_config(rpc: str, metadata: str):

tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py

@@ -30,6 +30,7 @@ DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
 RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
 XdsTestClient = client_app.XdsTestClient
 ExpectedResult = xds_url_map_testcase.ExpectedResult
+_Lang = skips.Lang
 
 logger = logging.getLogger(__name__)
 flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -62,15 +63,21 @@ def _build_retry_route_rule(retryConditions, num_retries):
     }
 
 
+def _is_supported(config: skips.TestConfig) -> bool:
+    # Per "Retry" in
+    # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+    if config.client_lang in _Lang.CPP | _Lang.JAVA | _Lang.PYTHON:
+        return not config.version_lt('v1.40.x')
+    elif config.client_lang == _Lang.GO:
+        return not config.version_lt('v1.41.x')
+    return True
+
+
 class TestRetryUpTo3AttemptsAndFail(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
     def is_supported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'java', 'python']:
-            return config.version_ge('v1.40.x')
-        elif config.client_lang == 'go':
-            return config.version_ge('v1.41.x')
-        return False
+        return _is_supported(config)
 
     @staticmethod
     def url_map_change(
@@ -111,11 +118,7 @@ class TestRetryUpTo4AttemptsAndSucceed(xds_url_map_testcase.XdsUrlMapTestCase):
 
     @staticmethod
     def is_supported(config: skips.TestConfig) -> bool:
-        if config.client_lang in ['cpp', 'java', 'python']:
-            return config.version_ge('v1.40.x')
-        elif config.client_lang == 'go':
-            return config.version_ge('v1.41.x')
-        return False
+        return _is_supported(config)
 
     @staticmethod
     def url_map_change(