Appnet schema changes (#27874)

* Add back references and scope field

* Set scope in router

* Reverse order of cleanup

* Add router_scope flag

* Use router_scope flag to create Router

* I apparently don't know how to brain

* Yapf

* Yeah, that can't be the default

* Remove debug print

* Remove impossible todos

* And another

* Switch from router-scope to config-scope

* Implement schema changes

* Use backend service URL

* Use CLH reference format to backend service

* I am an idiot

* *internal screaming*

* Try project number

* Why is this all awful

* Go back to trying project name

* Try cleaning things up

* Agh

* Address review comments

* Remove superfluous Optional type

tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py

@@ -69,6 +69,7 @@ def main(argv):
         gcp_service_account=gcp_service_account,
         xds_server_uri=xds_flags.XDS_SERVER_URI.value,
         network=xds_flags.NETWORK.value,
+        config_scope=xds_flags.ROUTER_SCOPE.value,
         stats_port=xds_flags.CLIENT_PORT.value,
         reuse_namespace=_REUSE_NAMESPACE.value)
 

tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py

@@ -60,6 +60,7 @@ class Router:
     name: str
     url: str
     type: str
+    scope: Optional[str]
     network: Optional[str]
     routes: Optional[List[str]]
 
@@ -69,6 +70,7 @@ class Router:
             name=name,
             url=d["name"],
             type=d["type"],
+            scope=d.get("scope"),
             network=d.get("network"),
             routes=list(d["routes"]) if "routes" in d else None,
         )
@@ -136,27 +138,30 @@ class GrpcRoute:
 
     @dataclasses.dataclass(frozen=True)
     class RouteAction:
-        destination: Optional['Destination']
+        destinations: List['Destination']
         drop: Optional[int]
 
         @classmethod
         def from_response(cls, d: Dict[str, Any]) -> 'RouteAction':
+            destinations = [
+                Destination.from_response(dest) for dest in d["destinations"]
+            ] if "destinations" in d else []
             return cls(
-                destination=Destination.from_response(d["destination"])
-                if "destination" in d else None,
+                destinations=destinations,
                 drop=d.get("drop"),
             )
 
     @dataclasses.dataclass(frozen=True)
     class RouteRule:
-        match: Optional['RouteMatch']
+        matches: List['RouteMatch']
         action: 'RouteAction'
 
         @classmethod
         def from_response(cls, d: Dict[str, Any]) -> 'RouteRule':
+            matches = [RouteMatch.from_response(m) for m in d["matches"]
+                      ] if "matches" in d else []
             return cls(
-                match=RouteMatch.from_response(d["match"])
-                if "match" in d else "",
+                matches=matches,
                 action=RouteAction.from_response(d["action"]),
             )
 
@@ -164,6 +169,7 @@ class GrpcRoute:
     url: str
     hostnames: Tuple[str]
     rules: Tuple['RouteRule']
+    routers: Optional[Tuple[str]]
 
     @classmethod
     def from_response(cls, name: str, d: Dict[str, Any]) -> 'RouteRule':
@@ -172,6 +178,7 @@ class GrpcRoute:
             url=d["name"],
             hostnames=tuple(d["hostnames"]),
             rules=tuple(d["rules"]),
+            routers=None if d.get("routers") is None else tuple(d["routers"]),
         )
 
 

tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py

@@ -556,6 +556,7 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager):
                  project: str,
                  *,
                  resource_prefix: str,
+                 config_scope: str,
                  resource_suffix: Optional[str] = None,
                  network: str = 'default',
                  compute_api_version: str = 'v1'):
@@ -566,6 +567,8 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager):
                          network=network,
                          compute_api_version=compute_api_version)
 
+        self.config_scope = config_scope
+
         # API
         self.netsvc = _NetworkServicesV1Alpha1(gcp_api_manager, project)
 
@@ -578,8 +581,7 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager):
         logger.info("Creating Router %s", name)
         body = {
             "type": "PROXYLESS_GRPC",
-            "routes": [self.grpc_route.url],
-            "network": "default",
+            "scope": self.config_scope,
         }
         resource = self.netsvc.create_router(name, body)
         self.router = self.netsvc.get_router(name)
@@ -599,14 +601,17 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager):
 
     def create_grpc_route(self, src_host: str, src_port: int) -> GcpResource:
         host = f'{src_host}:{src_port}'
+        service_name = self.netsvc.resource_full_name(self.backend_service.name,
+                                                      "backendServices")
         body = {
+            "routers": [self.router.url],
             "hostnames":
                 host,
             "rules": [{
                 "action": {
-                    "destination": {
-                        "serviceName": self.backend_service.name
-                    }
+                    "destinations": [{
+                        "serviceName": service_name
+                    }]
                 }
             }],
         }
@@ -637,8 +642,8 @@ class TrafficDirectorAppNetManager(TrafficDirectorManager):
         self.grpc_route = None
 
     def cleanup(self, *, force=False):
-        self.delete_router(force=force)
         self.delete_grpc_route(force=force)
+        self.delete_router(force=force)
         super().cleanup(force=force)
 
 

tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py

@@ -238,6 +238,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner):
                  gcp_service_account: str,
                  xds_server_uri=None,
                  network='default',
+                 config_scope=None,
                  service_account_name=None,
                  stats_port=8079,
                  deployment_template='client.deployment.yaml',
@@ -256,6 +257,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner):
         self.td_bootstrap_image = td_bootstrap_image
         self.xds_server_uri = xds_server_uri
         self.network = network
+        self.config_scope = config_scope
         self.deployment_template = deployment_template
         self.debug_use_port_forwarding = debug_use_port_forwarding
         self.enable_workload_identity = enable_workload_identity
@@ -327,6 +329,7 @@ class KubernetesClientRunner(base_runner.KubernetesBaseRunner):
             td_bootstrap_image=self.td_bootstrap_image,
             xds_server_uri=self.xds_server_uri,
             network=self.network,
+            config_scope=self.config_scope,
             stats_port=self.stats_port,
             server_target=server_target,
             rpc=rpc,

tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py

@@ -36,6 +36,10 @@ RESOURCE_SUFFIX = flags.DEFINE_string(
 NETWORK = flags.DEFINE_string("network",
                               default="default",
                               help="GCP Network ID")
+ROUTER_SCOPE = flags.DEFINE_string(
+    "config_scope",
+    default=None,
+    help="Scope specified in router if using AppNet APIs")
 COMPUTE_API_VERSION = flags.DEFINE_string(
     "compute_api_version",
     default='v1',

tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py

@@ -87,6 +87,7 @@ class XdsKubernetesTestCase(absltest.TestCase, metaclass=abc.ABCMeta):
         # GCP
         cls.project: str = xds_flags.PROJECT.value
         cls.network: str = xds_flags.NETWORK.value
+        cls.config_scope: str = xds_flags.ROUTER_SCOPE.value
         cls.gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
         cls.td_bootstrap_image = xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value
         cls.xds_server_uri = xds_flags.XDS_SERVER_URI.value
@@ -418,6 +419,7 @@ class RegularXdsKubernetesTestCase(XdsKubernetesTestCase):
             gcp_service_account=self.gcp_service_account,
             xds_server_uri=self.xds_server_uri,
             network=self.network,
+            config_scope=self.config_scope,
             debug_use_port_forwarding=self.debug_use_port_forwarding,
             enable_workload_identity=self.enable_workload_identity,
             stats_port=self.client_port,
@@ -457,6 +459,7 @@ class AppNetXdsKubernetesTestCase(RegularXdsKubernetesTestCase):
             resource_prefix=self.resource_prefix,
             resource_suffix=self.resource_suffix,
             network=self.network,
+            config_scope=self.config_scope,
             compute_api_version=self.compute_api_version)
 
 
@@ -518,6 +521,7 @@ class SecurityXdsKubernetesTestCase(XdsKubernetesTestCase):
             gcp_service_account=self.gcp_service_account,
             xds_server_uri=self.xds_server_uri,
             network=self.network,
+            config_scope=self.config_scope,
             deployment_template='client-secure.deployment.yaml',
             stats_port=self.client_port,
             reuse_namespace=self.server_namespace == self.client_namespace,

tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml

@@ -62,6 +62,9 @@ spec:
             % if xds_server_uri:
             - "--xds-server-uri=${xds_server_uri}"
             % endif
+            % if config_scope:
+            - "--config-scope-experimental=${config_scope_experimental}"
+            % endif
             - "--include-v3-features-experimental"
             - "--include-psm-security-experimental"
           resources:

tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml

@@ -62,6 +62,9 @@ spec:
             % if xds_server_uri:
             - "--xds-server-uri=${xds_server_uri}"
             % endif
+            % if config_scope:
+            - "--config-scope-experimental=${config_scope}"
+            % endif
           resources:
             limits:
               cpu: 100m

tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py

@@ -34,13 +34,13 @@ class AppNetTest(xds_k8s_testcase.AppNetXdsKubernetesTestCase):
         with self.subTest('1_create_backend_service'):
             self.td.create_backend_service()
 
-        with self.subTest('2_create_grpc_route'):
+        with self.subTest('2_create_router'):
+            self.td.create_router()
+
+        with self.subTest('3_create_grpc_route'):
             self.td.create_grpc_route(self.server_xds_host,
                                       self.server_xds_port)
 
-        with self.subTest('3_create_router'):
-            self.td.create_router()
-
         with self.subTest('4_start_test_server'):
             test_server: _XdsTestServer = self.startTestServers(
                 replica_count=1)[0]