From b5217edca732f42ce15d83ae7d9f0be1e27446d7 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 4 Nov 2021 10:55:54 -0700 Subject: [PATCH] Appnet schema changes (#27874) * Add back references and scope field * Set scope in router * Reverse order of cleanup * Add router_scope flag * Use router_scope flag to create Router * I apparently don't know how to brain * Yapf * Yeah, that can't be the default * Remove debug print * Remove impossible todos * And another * Switch from router-scope to config-scope * Implement schema changes * Use backend service URL * Use CLH reference format to backend service * I am an idiot * *internal screaming* * Try project number * Why is this all awful * Go back to trying project name * Try cleaning things up * Agh * Address review comments * Remove superfluous Optional type --- .../bin/run_test_client.py | 1 + .../infrastructure/gcp/network_services.py | 19 +++++++++++++------ .../infrastructure/traffic_director.py | 17 +++++++++++------ .../framework/test_app/client_app.py | 3 +++ .../framework/xds_flags.py | 4 ++++ .../framework/xds_k8s_testcase.py | 4 ++++ .../client-secure.deployment.yaml | 3 +++ .../client.deployment.yaml | 3 +++ .../xds_k8s_test_driver/tests/app_net_test.py | 8 ++++---- 9 files changed, 46 insertions(+), 16 deletions(-) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py b/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py index 1e600935d5e..ca194290fae 100755 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py +++ b/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py @@ -69,6 +69,7 @@ def main(argv): gcp_service_account=gcp_service_account, xds_server_uri=xds_flags.XDS_SERVER_URI.value, network=xds_flags.NETWORK.value, + config_scope=xds_flags.ROUTER_SCOPE.value, stats_port=xds_flags.CLIENT_PORT.value, reuse_namespace=_REUSE_NAMESPACE.value) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py index 59cd5655d3d..eb9abe6ee96 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py @@ -60,6 +60,7 @@ class Router: name: str url: str type: str + scope: Optional[str] network: Optional[str] routes: Optional[List[str]] @@ -69,6 +70,7 @@ class Router: name=name, url=d["name"], type=d["type"], + scope=d.get("scope"), network=d.get("network"), routes=list(d["routes"]) if "routes" in d else None, ) @@ -136,27 +138,30 @@ class GrpcRoute: @dataclasses.dataclass(frozen=True) class RouteAction: - destination: Optional['Destination'] + destinations: List['Destination'] drop: Optional[int] @classmethod def from_response(cls, d: Dict[str, Any]) -> 'RouteAction': + destinations = [ + Destination.from_response(dest) for dest in d["destinations"] + ] if "destinations" in d else [] return cls( - destination=Destination.from_response(d["destination"]) - if "destination" in d else None, + destinations=destinations, drop=d.get("drop"), ) @dataclasses.dataclass(frozen=True) class RouteRule: - match: Optional['RouteMatch'] + matches: List['RouteMatch'] action: 'RouteAction' @classmethod def from_response(cls, d: Dict[str, Any]) -> 'RouteRule': + matches = [RouteMatch.from_response(m) for m in d["matches"] + ] if "matches" in d else [] return cls( - match=RouteMatch.from_response(d["match"]) - if "match" in d else "", + matches=matches, action=RouteAction.from_response(d["action"]), ) @@ -164,6 +169,7 @@ class GrpcRoute: url: str hostnames: Tuple[str] rules: Tuple['RouteRule'] + routers: Optional[Tuple[str]] @classmethod def from_response(cls, name: str, d: Dict[str, Any]) -> 'RouteRule': @@ -172,6 +178,7 @@ class GrpcRoute: url=d["name"], hostnames=tuple(d["hostnames"]), rules=tuple(d["rules"]), + routers=None if d.get("routers") is None else tuple(d["routers"]), ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py index 9611bc21da9..f71bc27ffee 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py @@ -556,6 +556,7 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager): project: str, *, resource_prefix: str, + config_scope: str, resource_suffix: Optional[str] = None, network: str = 'default', compute_api_version: str = 'v1'): @@ -566,6 +567,8 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager): network=network, compute_api_version=compute_api_version) + self.config_scope = config_scope + # API self.netsvc = _NetworkServicesV1Alpha1(gcp_api_manager, project) @@ -578,8 +581,7 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager): logger.info("Creating Router %s", name) body = { "type": "PROXYLESS_GRPC", - "routes": [self.grpc_route.url], - "network": "default", + "scope": self.config_scope, } resource = self.netsvc.create_router(name, body) self.router = self.netsvc.get_router(name) @@ -599,14 +601,17 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager): def create_grpc_route(self, src_host: str, src_port: int) -> GcpResource: host = f'{src_host}:{src_port}' + service_name = self.netsvc.resource_full_name(self.backend_service.name, + "backendServices") body = { + "routers": [self.router.url], "hostnames": host, "rules": [{ "action": { - "destination": { - "serviceName": self.backend_service.name - } + "destinations": [{ + "serviceName": service_name + }] } }], } @@ -637,8 +642,8 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager): self.grpc_route = None def cleanup(self, *, force=False): - self.delete_router(force=force) self.delete_grpc_route(force=force) + self.delete_router(force=force) super().cleanup(force=force) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py index 9645db93fd9..4ddfa3ebf55 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py @@ -238,6 +238,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner): gcp_service_account: str, xds_server_uri=None, network='default', + config_scope=None, service_account_name=None, stats_port=8079, deployment_template='client.deployment.yaml', @@ -256,6 +257,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner): self.td_bootstrap_image = td_bootstrap_image self.xds_server_uri = xds_server_uri self.network = network + self.config_scope = config_scope self.deployment_template = deployment_template self.debug_use_port_forwarding = debug_use_port_forwarding self.enable_workload_identity = enable_workload_identity @@ -327,6 +329,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner): td_bootstrap_image=self.td_bootstrap_image, xds_server_uri=self.xds_server_uri, network=self.network, + config_scope=self.config_scope, stats_port=self.stats_port, server_target=server_target, rpc=rpc, diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py index 578f4e3fe6d..4b3408fe1ee 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py @@ -36,6 +36,10 @@ RESOURCE_SUFFIX = flags.DEFINE_string( NETWORK = flags.DEFINE_string("network", default="default", help="GCP Network ID") +ROUTER_SCOPE = flags.DEFINE_string( + "config_scope", + default=None, + help="Scope specified in router if using AppNet APIs") COMPUTE_API_VERSION = flags.DEFINE_string( "compute_api_version", default='v1', diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py index 8c8e83d2676..b9e6f25a107 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py @@ -87,6 +87,7 @@ class XdsKubernetesTestCase(absltest.TestCase, metaclass=abc.ABCMeta): # GCP cls.project: str = xds_flags.PROJECT.value cls.network: str = xds_flags.NETWORK.value + cls.config_scope: str = xds_flags.ROUTER_SCOPE.value cls.gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value cls.td_bootstrap_image = xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value cls.xds_server_uri = xds_flags.XDS_SERVER_URI.value @@ -418,6 +419,7 @@ class RegularXdsKubernetesTestCase(XdsKubernetesTestCase): gcp_service_account=self.gcp_service_account, xds_server_uri=self.xds_server_uri, network=self.network, + config_scope=self.config_scope, debug_use_port_forwarding=self.debug_use_port_forwarding, enable_workload_identity=self.enable_workload_identity, stats_port=self.client_port, @@ -457,6 +459,7 @@ class AppNetXdsKubernetesTestCase(RegularXdsKubernetesTestCase): resource_prefix=self.resource_prefix, resource_suffix=self.resource_suffix, network=self.network, + config_scope=self.config_scope, compute_api_version=self.compute_api_version) @@ -518,6 +521,7 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase): gcp_service_account=self.gcp_service_account, xds_server_uri=self.xds_server_uri, network=self.network, + config_scope=self.config_scope, deployment_template='client-secure.deployment.yaml', stats_port=self.client_port, reuse_namespace=self.server_namespace == self.client_namespace, diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml index 6db4192e90d..6e2fdae14f3 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml @@ -62,6 +62,9 @@ spec: % if xds_server_uri: - "--xds-server-uri=${xds_server_uri}" % endif + % if config_scope: + - "--config-scope-experimental=${config_scope_experimental}" + % endif - "--include-v3-features-experimental" - "--include-psm-security-experimental" resources: diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml index 31ef54fe6ea..72ade8b1397 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml @@ -62,6 +62,9 @@ spec: % if xds_server_uri: - "--xds-server-uri=${xds_server_uri}" % endif + % if config_scope: + - "--config-scope-experimental=${config_scope}" + % endif resources: limits: cpu: 100m diff --git a/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py b/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py index 8c31cfb35e5..0ffbb4c1ef1 100644 --- a/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py +++ b/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py @@ -34,13 +34,13 @@ class AppNetTest(xds_k8s_testcase.AppNetXdsKubernetesTestCase): with self.subTest('1_create_backend_service'): self.td.create_backend_service() - with self.subTest('2_create_grpc_route'): + with self.subTest('2_create_router'): + self.td.create_router() + + with self.subTest('3_create_grpc_route'): self.td.create_grpc_route(self.server_xds_host, self.server_xds_port) - with self.subTest('3_create_router'): - self.td.create_router() - with self.subTest('4_start_test_server'): test_server: _XdsTestServer = self.startTestServers( replica_count=1)[0]