diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py
index efea6b07726..3cf52ac9379 100644
--- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py
+++ b/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py
@@ -13,8 +13,10 @@
# limitations under the License.
"""The classes and predicates to assist validate test config for test cases."""
from dataclasses import dataclass
+import enum
+import logging
import re
-from typing import Callable
+from typing import Callable, Optional
import unittest
from packaging import version as pkg_version
@@ -22,35 +24,75 @@ from packaging import version as pkg_version
from framework import xds_flags
from framework import xds_k8s_flags
+logger = logging.getLogger(__name__)
-def _get_lang(image_name: str) -> str:
- return re.search(r'/(\w+)-(client|server):', image_name).group(1)
+class Lang(enum.Flag):
+ UNKNOWN = enum.auto()
+ CPP = enum.auto()
+ GO = enum.auto()
+ JAVA = enum.auto()
+ PYTHON = enum.auto()
+ NODE = enum.auto()
-def _parse_version(s: str) -> pkg_version.Version:
- if s.endswith(".x"):
- s = s[:-2]
- return pkg_version.Version(s)
+ def __str__(self):
+ return str(self.name).lower()
+
+ @classmethod
+ def from_string(cls, lang: str):
+ try:
+ return cls[lang.upper()]
+ except KeyError:
+ return cls.UNKNOWN
@dataclass
class TestConfig:
"""Describes the config for the test suite."""
- client_lang: str
- server_lang: str
- version: str
+ client_lang: Lang
+ server_lang: Lang
+ version: Optional[str]
- def version_ge(self, another: str) -> bool:
+ def version_gte(self, another: str) -> bool:
"""Returns a bool for whether the version is >= another one.
A version is greater than or equal to another version means its version
number is greater than or equal to another version's number. Version
- "master" is always considered latest. E.g., master >= v1.41.x >= v1.40.x
- >= v1.9.x.
+ "master" is always considered latest.
+ E.g., master >= v1.41.x >= v1.40.x >= v1.9.x.
+
+ Unspecified version is treated as 'master', but isn't explicitly set.
"""
- if self.version == 'master':
+ if self.version == 'master' or self.version is None:
return True
- return _parse_version(self.version) >= _parse_version(another)
+ return self._parse_version(self.version) >= self._parse_version(another)
+
+ def version_lt(self, another: str) -> bool:
+ """Returns a bool for whether the version is < another one.
+
+ Version "master" is always considered latest.
+ E.g., v1.9.x < v1.40.x < v1.41.x < master.
+
+ Unspecified version is treated as 'master', but isn't explicitly set.
+ """
+ if self.version == 'master' or self.version is None:
+ return False
+ return self._parse_version(self.version) < self._parse_version(another)
+
+ def __str__(self):
+ return (f"TestConfig(client_lang='{self.client_lang}', "
+ f"server_lang='{self.server_lang}', version={self.version!r})")
+
+ @staticmethod
+ def _parse_version(s: str) -> pkg_version.Version:
+ if s.endswith(".x"):
+ s = s[:-2]
+ return pkg_version.Version(s)
+
+
+def _get_lang(image_name: str) -> Lang:
+ return Lang.from_string(
+ re.search(r'/(\w+)-(client|server):', image_name).group(1))
def evaluate_test_config(check: Callable[[TestConfig], bool]) -> None:
@@ -64,4 +106,7 @@ def evaluate_test_config(check: Callable[[TestConfig], bool]) -> None:
server_lang=_get_lang(xds_k8s_flags.SERVER_IMAGE.value),
version=xds_flags.TESTING_VERSION.value)
if not check(test_config):
+ logger.info('Skipping %s', test_config)
raise unittest.SkipTest(f'Unsupported test config: {test_config}')
+
+ logger.info('Detected language and version: %s', test_config)
diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py
index e6412da4c67..2b411b1643c 100644
--- a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py
+++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py
@@ -116,7 +116,7 @@ CLIENT_PORT = flags.DEFINE_integer(
# Testing metadata
TESTING_VERSION = flags.DEFINE_string(
"testing_version",
- default="master",
+ default=None,
help="The testing gRPC version branch name. Like master, v1.41.x, v1.37.x")
FORCE_CLEANUP = flags.DEFINE_bool(
diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py
index 7b7561580ca..fcf394a5da7 100644
--- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py
+++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py
@@ -85,7 +85,7 @@ class XdsKubernetesBaseTestCase(absltest.TestCase):
td: TrafficDirectorManager
@staticmethod
- def isSupported(config: skips.TestConfig) -> bool:
+ def is_supported(config: skips.TestConfig) -> bool:
"""Overridden by the test class to decide if the config is supported.
Returns:
@@ -104,7 +104,7 @@ class XdsKubernetesBaseTestCase(absltest.TestCase):
# Raises unittest.SkipTest if given client/server/version does not
# support current test case.
- skips.evaluate_test_config(cls.isSupported)
+ skips.evaluate_test_config(cls.is_supported)
# GCP
cls.project: str = xds_flags.PROJECT.value
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py b/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py
index eef8f15e26b..27b18a825a1 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py
@@ -31,6 +31,7 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
_XdsTestServer = xds_k8s_testcase.XdsTestServer
_XdsTestClient = xds_k8s_testcase.XdsTestClient
_ChannelzChannelState = grpc_channelz.ChannelState
+_Lang = skips.Lang
# Testing consts
_TEST_AFFINITY_METADATA_KEY = 'xds_md'
@@ -43,10 +44,12 @@ _RPC_COUNT = 100
class AffinityTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
@staticmethod
- def isSupported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'java', 'python', 'go']:
- return config.version_ge('v1.40.x')
- return False
+ def is_supported(config: skips.TestConfig) -> bool:
+ if config.client_lang in (_Lang.CPP | _Lang.GO | _Lang.JAVA |
+ _Lang.PYTHON):
+ # Versions prior to v1.40.x don't support Affinity.
+ return not config.version_lt('v1.40.x')
+ return True
def test_affinity(self) -> None: # pylint: disable=too-many-statements
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py b/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py
index 47efa4399ab..ffb474fa12c 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py
@@ -29,6 +29,7 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
_XdsTestServer = xds_k8s_testcase.XdsTestServer
_XdsTestClient = xds_k8s_testcase.XdsTestClient
_SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode
+_Lang = skips.Lang
# The client generates QPS even when it is still loading information from xDS.
# Once it finally connects there will be an outpouring of the bufferred RPCs and
@@ -47,12 +48,14 @@ class AuthzTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
}
@staticmethod
- def isSupported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'python']:
- return config.version_ge('v1.44.x')
- elif config.client_lang in ['java', 'go']:
- return config.version_ge('v1.42.x')
- return False
+ def is_supported(config: skips.TestConfig) -> bool:
+ # Per "Authorization (RBAC)" in
+ # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+ if config.client_lang in _Lang.CPP | _Lang.PYTHON:
+ return not config.version_lt('v1.44.x')
+ elif config.client_lang in _Lang.GO | _Lang.JAVA:
+ return not config.version_lt('v1.42.x')
+ return True
def setUp(self):
super().setUp()
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/security_test.py b/tools/run_tests/xds_k8s_test_driver/tests/security_test.py
index 0d70f3083df..8ef0edc4ea7 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/security_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/security_test.py
@@ -27,15 +27,19 @@ flags.adopt_module_key_flags(xds_k8s_testcase)
_XdsTestServer = xds_k8s_testcase.XdsTestServer
_XdsTestClient = xds_k8s_testcase.XdsTestClient
_SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode
+_Lang = skips.Lang
class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase):
@staticmethod
- def isSupported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'python', 'go']:
- return config.version_ge('v1.41.x')
- return False
+ def is_supported(config: skips.TestConfig) -> bool:
+ if config.client_lang in (_Lang.CPP | _Lang.GO | _Lang.JAVA |
+ _Lang.PYTHON):
+ # Versions prior to v1.41.x don't support PSM Security.
+ # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+ return not config.version_lt('v1.41.x')
+ return True
def test_mtls(self):
"""mTLS test.
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py b/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py
index b6292037047..abe537323df 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py
@@ -38,11 +38,11 @@ _NUM_CLIENTS = 3
class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
@staticmethod
- def isSupported(config: skips.TestConfig) -> bool:
+ def is_supported(config: skips.TestConfig) -> bool:
# Subsetting is an experimental feature where most work is done on the
# server-side. We limit it to only run on master branch to save
# resources.
- return config.version_ge('master')
+ return config.version_gte('master')
def test_subsetting_basic(self) -> None:
with self.subTest('00_create_health_check'):
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py
index fe079bbc5e3..6a29f227856 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py
@@ -31,6 +31,7 @@ DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall
XdsTestClient = client_app.XdsTestClient
+_Lang = skips.Lang
logger = logging.getLogger(__name__)
flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -52,15 +53,25 @@ _TEST_METADATA = (
_ChannelzChannelState = grpc_channelz.ChannelState
+def _is_supported(config: skips.TestConfig) -> bool:
+ # Per "Ring hash" in
+ # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+ if config.client_lang in _Lang.CPP | _Lang.JAVA:
+ return not config.version_lt('v1.40.x')
+ elif config.client_lang == _Lang.GO:
+ return not config.version_lt('v1.41.x')
+ elif config.client_lang == _Lang.PYTHON:
+ # TODO(https://github.com/grpc/grpc/issues/27430): supported after
+ # the issue is fixed.
+ return False
+ return True
+
+
class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase):
@staticmethod
def is_supported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'java']:
- return config.version_ge('v1.40.x')
- if config.client_lang in ['go']:
- return config.version_ge('v1.41.x')
- return False
+ return _is_supported(config)
@staticmethod
def client_init_config(rpc: str, metadata: str):
@@ -125,11 +136,7 @@ class TestHeaderBasedAffinityMultipleHeaders(
@staticmethod
def is_supported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'java']:
- return config.version_ge('v1.40.x')
- if config.client_lang in ['go']:
- return config.version_ge('v1.41.x')
- return False
+ return _is_supported(config)
@staticmethod
def client_init_config(rpc: str, metadata: str):
diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py
index 8929e99c1b2..89fb34ca07d 100644
--- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py
+++ b/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py
@@ -30,6 +30,7 @@ DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig
RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall
XdsTestClient = client_app.XdsTestClient
ExpectedResult = xds_url_map_testcase.ExpectedResult
+_Lang = skips.Lang
logger = logging.getLogger(__name__)
flags.adopt_module_key_flags(xds_url_map_testcase)
@@ -62,15 +63,21 @@ def _build_retry_route_rule(retryConditions, num_retries):
}
+def _is_supported(config: skips.TestConfig) -> bool:
+ # Per "Retry" in
+ # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md
+ if config.client_lang in _Lang.CPP | _Lang.JAVA | _Lang.PYTHON:
+ return not config.version_lt('v1.40.x')
+ elif config.client_lang == _Lang.GO:
+ return not config.version_lt('v1.41.x')
+ return True
+
+
class TestRetryUpTo3AttemptsAndFail(xds_url_map_testcase.XdsUrlMapTestCase):
@staticmethod
def is_supported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'java', 'python']:
- return config.version_ge('v1.40.x')
- elif config.client_lang == 'go':
- return config.version_ge('v1.41.x')
- return False
+ return _is_supported(config)
@staticmethod
def url_map_change(
@@ -111,11 +118,7 @@ class TestRetryUpTo4AttemptsAndSucceed(xds_url_map_testcase.XdsUrlMapTestCase):
@staticmethod
def is_supported(config: skips.TestConfig) -> bool:
- if config.client_lang in ['cpp', 'java', 'python']:
- return config.version_ge('v1.40.x')
- elif config.client_lang == 'go':
- return config.version_ge('v1.41.x')
- return False
+ return _is_supported(config)
@staticmethod
def url_map_change(