mirror of https://github.com/grpc/grpc.git
[PSM Interop] Initial support for GAMMA tests (#34151)
Adds initial support for K8s [GAMMA](https://gateway-api.sigs.k8s.io/concepts/gamma/) (Gateway API for Service Mesh) initiative. - Add framework support for loading CRD-based APIs using k8s python dynamic client - Add basic mesh baseline test (aka ping-pong) using GAMMA setup - Implement initial framework changes needed to run PSM tests on GAMMA-enabled cluster using [TDMesh](https://cloud.google.com/traffic-director/docs/gke-gateway-overview#gateway-api) and GRPCRoute. Based on https://github.com/grpc/grpc/pull/33504.pull/34166/head
Sergii Tkachenko
2 years ago
committed by
GitHub
parent
a6689e6444
commit
2c5abd316d
19 changed files with 874 additions and 74 deletions
@ -0,0 +1,22 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import framework.infrastructure.traffic_director as td_base |
||||
|
||||
|
||||
# TODO(sergiitk): [GAMMA] make a TD-manager-less base test case. |
||||
class TrafficDirectorGammaManager(td_base.TrafficDirectorManager): |
||||
"""Gamma.""" |
||||
|
||||
def cleanup(self, *, force=False): # pylint: disable=unused-argument |
||||
return True |
||||
@ -0,0 +1,242 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
""" |
||||
Run xDS Test Client on Kubernetes using Gamma |
||||
""" |
||||
import logging |
||||
from typing import List, Optional |
||||
|
||||
from framework.infrastructure import gcp |
||||
from framework.infrastructure import k8s |
||||
from framework.test_app.runners.k8s import k8s_xds_server_runner |
||||
from framework.test_app.server_app import XdsTestServer |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner |
||||
|
||||
|
||||
class GammaServerRunner(KubernetesServerRunner): |
||||
# Mutable state. |
||||
mesh: Optional[k8s.GammaMesh] = None |
||||
route: Optional[k8s.GammaGrpcRoute] = None |
||||
|
||||
# Mesh |
||||
server_xds_host: str |
||||
mesh_name: str |
||||
route_name: str |
||||
|
||||
def __init__( |
||||
self, |
||||
k8s_namespace: k8s.KubernetesNamespace, |
||||
*, |
||||
mesh_name: str, |
||||
server_xds_host: str, |
||||
deployment_name: str, |
||||
image_name: str, |
||||
td_bootstrap_image: str, |
||||
network: str = "default", |
||||
xds_server_uri: Optional[str] = None, |
||||
gcp_api_manager: gcp.api.GcpApiManager, |
||||
gcp_project: str, |
||||
gcp_service_account: str, |
||||
service_account_name: Optional[str] = None, |
||||
service_name: Optional[str] = None, |
||||
route_name: Optional[str] = None, |
||||
neg_name: Optional[str] = None, |
||||
deployment_template: str = "server.deployment.yaml", |
||||
service_account_template: str = "service-account.yaml", |
||||
service_template: str = "gamma/service.yaml", |
||||
reuse_service: bool = False, |
||||
reuse_namespace: bool = False, |
||||
namespace_template: Optional[str] = None, |
||||
debug_use_port_forwarding: bool = False, |
||||
enable_workload_identity: bool = True, |
||||
): |
||||
# pylint: disable=too-many-locals |
||||
super().__init__( |
||||
k8s_namespace, |
||||
deployment_name=deployment_name, |
||||
image_name=image_name, |
||||
td_bootstrap_image=td_bootstrap_image, |
||||
network=network, |
||||
xds_server_uri=xds_server_uri, |
||||
gcp_api_manager=gcp_api_manager, |
||||
gcp_project=gcp_project, |
||||
gcp_service_account=gcp_service_account, |
||||
service_account_name=service_account_name, |
||||
service_name=service_name, |
||||
neg_name=neg_name, |
||||
deployment_template=deployment_template, |
||||
service_account_template=service_account_template, |
||||
service_template=service_template, |
||||
reuse_service=reuse_service, |
||||
reuse_namespace=reuse_namespace, |
||||
namespace_template=namespace_template, |
||||
debug_use_port_forwarding=debug_use_port_forwarding, |
||||
enable_workload_identity=enable_workload_identity, |
||||
) |
||||
|
||||
self.server_xds_host = server_xds_host |
||||
self.mesh_name = mesh_name |
||||
self.route_name = route_name or f"route-{deployment_name}" |
||||
|
||||
def run( |
||||
self, |
||||
*, |
||||
test_port: int = KubernetesServerRunner.DEFAULT_TEST_PORT, |
||||
maintenance_port: Optional[int] = None, |
||||
secure_mode: bool = False, |
||||
replica_count: int = 1, |
||||
log_to_stdout: bool = False, |
||||
bootstrap_version: Optional[str] = None, |
||||
) -> List[XdsTestServer]: |
||||
if not maintenance_port: |
||||
maintenance_port = self._get_default_maintenance_port(secure_mode) |
||||
|
||||
logger.info( |
||||
( |
||||
'Deploying GAMMA xDS test server "%s" to k8s namespace %s:' |
||||
" test_port=%s maintenance_port=%s secure_mode=%s" |
||||
" replica_count=%s" |
||||
), |
||||
self.deployment_name, |
||||
self.k8s_namespace.name, |
||||
test_port, |
||||
maintenance_port, |
||||
False, |
||||
replica_count, |
||||
) |
||||
# super(k8s_base_runner.KubernetesBaseRunner, self).run() |
||||
|
||||
if self.reuse_namespace: |
||||
self.namespace = self._reuse_namespace() |
||||
if not self.namespace: |
||||
self.namespace = self._create_namespace( |
||||
self.namespace_template, namespace_name=self.k8s_namespace.name |
||||
) |
||||
|
||||
# Create gamma mesh. |
||||
# Note: this will be pre-provisioned per cluster. |
||||
self.mesh = self._create_gamma_mesh( |
||||
"gamma/tdmesh.yaml", |
||||
mesh_name=self.mesh_name, |
||||
namespace_name=self.k8s_namespace.name, |
||||
) |
||||
|
||||
# Reuse existing if requested, create a new deployment when missing. |
||||
# Useful for debugging to avoid NEG loosing relation to deleted service. |
||||
if self.reuse_service: |
||||
self.service = self._reuse_service(self.service_name) |
||||
if not self.service: |
||||
self.service = self._create_service( |
||||
self.service_template, |
||||
service_name=self.service_name, |
||||
namespace_name=self.k8s_namespace.name, |
||||
deployment_name=self.deployment_name, |
||||
neg_name=self.gcp_neg_name, |
||||
test_port=test_port, |
||||
) |
||||
|
||||
# Create the route. |
||||
self.route = self._create_gamma_route( |
||||
"gamma/route_grpc.yaml", |
||||
xds_server_uri=self.server_xds_host, |
||||
route_name=self.route_name, |
||||
mesh_name=self.mesh_name, |
||||
service_name=self.service_name, |
||||
namespace_name=self.k8s_namespace.name, |
||||
test_port=test_port, |
||||
) |
||||
|
||||
# Surprised this just works. |
||||
self._wait_service_neg(self.service_name, test_port) |
||||
|
||||
if self.enable_workload_identity: |
||||
# Allow Kubernetes service account to use the GCP service account |
||||
# identity. |
||||
self._grant_workload_identity_user( |
||||
gcp_iam=self.gcp_iam, |
||||
gcp_service_account=self.gcp_service_account, |
||||
service_account_name=self.service_account_name, |
||||
) |
||||
|
||||
# Create service account |
||||
self.service_account = self._create_service_account( |
||||
self.service_account_template, |
||||
service_account_name=self.service_account_name, |
||||
namespace_name=self.k8s_namespace.name, |
||||
gcp_service_account=self.gcp_service_account, |
||||
) |
||||
|
||||
# Always create a new deployment |
||||
self.deployment = self._create_deployment( |
||||
self.deployment_template, |
||||
deployment_name=self.deployment_name, |
||||
image_name=self.image_name, |
||||
namespace_name=self.k8s_namespace.name, |
||||
service_account_name=self.service_account_name, |
||||
td_bootstrap_image=self.td_bootstrap_image, |
||||
xds_server_uri=self.xds_server_uri, |
||||
network=self.network, |
||||
replica_count=replica_count, |
||||
test_port=test_port, |
||||
maintenance_port=maintenance_port, |
||||
secure_mode=secure_mode, |
||||
bootstrap_version=bootstrap_version, |
||||
) |
||||
|
||||
return self._make_servers_for_deployment( |
||||
replica_count, |
||||
test_port=test_port, |
||||
maintenance_port=maintenance_port, |
||||
log_to_stdout=log_to_stdout, |
||||
secure_mode=secure_mode, |
||||
) |
||||
|
||||
# pylint: disable=arguments-differ |
||||
def cleanup(self, *, force=False, force_namespace=False): |
||||
try: |
||||
if self.route or force: |
||||
self._delete_gamma_route(self.route_name) |
||||
self.route = None |
||||
|
||||
if self.mesh or force: |
||||
self._delete_gamma_mesh(self.mesh_name) |
||||
|
||||
if (self.service and not self.reuse_service) or force: |
||||
self._delete_service(self.service_name) |
||||
self.service = None |
||||
|
||||
if self.deployment or force: |
||||
self._delete_deployment(self.deployment_name) |
||||
self.deployment = None |
||||
|
||||
if self.enable_workload_identity and ( |
||||
self.service_account or force |
||||
): |
||||
self._revoke_workload_identity_user( |
||||
gcp_iam=self.gcp_iam, |
||||
gcp_service_account=self.gcp_service_account, |
||||
service_account_name=self.service_account_name, |
||||
) |
||||
self._delete_service_account(self.service_account_name) |
||||
self.service_account = None |
||||
|
||||
self._cleanup_namespace(force=(force_namespace and force)) |
||||
finally: |
||||
self._stop() |
||||
|
||||
# pylint: enable=arguments-differ |
||||
@ -0,0 +1,118 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
|
||||
from framework.infrastructure import k8s |
||||
import framework.infrastructure.traffic_director_gamma as td_gamma |
||||
from framework.test_app import client_app |
||||
from framework.test_app import server_app |
||||
from framework.test_app.runners.k8s import gamma_server_runner |
||||
from framework.test_app.runners.k8s import k8s_xds_client_runner |
||||
import framework.xds_k8s_testcase as xds_k8s_testcase |
||||
|
||||
GammaServerRunner = gamma_server_runner.GammaServerRunner |
||||
KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner |
||||
XdsTestClient = client_app.XdsTestClient |
||||
XdsTestServer = server_app.XdsTestServer |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
|
||||
|
||||
# TODO(sergiitk): [GAMMA] Move into framework/test_cases |
||||
class GammaXdsKubernetesTestCase(xds_k8s_testcase.RegularXdsKubernetesTestCase): |
||||
server_runner: GammaServerRunner |
||||
mesh_name: str |
||||
mesh_name_td: str |
||||
|
||||
def setUp(self): |
||||
"""Hook method for setting up the test fixture before exercising it.""" |
||||
# TODO(sergiitk): [GAMMA] Remove when refactored to be TD-manager-less. |
||||
# pylint: disable=bad-super-call |
||||
# Skips RegularXdsKubernetesTestCase and IsolatedXdsKubernetesTestCase |
||||
# and calls setUp on XdsKubernetesBaseTestCase. |
||||
# IsolatedXdsKubernetesTestCase randomizes server_xds_port when it's 0, |
||||
# and in GAMMA we always need it unset. |
||||
# Calls XdsKubernetesBaseTestCase.setUp(): |
||||
super(xds_k8s_testcase.IsolatedXdsKubernetesTestCase, self).setUp() |
||||
# pylint: enable=bad-super-call |
||||
|
||||
# Random suffix per test. |
||||
self.createRandomSuffix() |
||||
|
||||
# TODO(sergiitk): [GAMMA] Make a TD-manager-less base test case |
||||
# TD Manager |
||||
self.td = self.initTrafficDirectorManager() |
||||
|
||||
# Generate unique mesh name too. |
||||
self.mesh_name = f"{self.resource_prefix}-mesh-{self.resource_suffix}" |
||||
self.mesh_name_td = f"gketd-{self.mesh_name}" |
||||
|
||||
# The gamma mesh doesn't use the port. |
||||
self.server_xds_host = f"{self.server_xds_host}-{self.resource_suffix}" |
||||
self.server_xds_port = None |
||||
|
||||
# Test Server runner |
||||
self.server_namespace = GammaServerRunner.make_namespace_name( |
||||
self.resource_prefix, self.resource_suffix |
||||
) |
||||
self.server_runner = self.initKubernetesServerRunner() |
||||
|
||||
# Test Client runner |
||||
self.client_namespace = KubernetesClientRunner.make_namespace_name( |
||||
self.resource_prefix, self.resource_suffix |
||||
) |
||||
self.client_runner = self.initKubernetesClientRunner() |
||||
|
||||
# Cleanup. |
||||
self.force_cleanup = True |
||||
self.force_cleanup_namespace = True |
||||
|
||||
# TODO(sergiitk): [GAMMA] Make a TD-manager-less base test case |
||||
def initTrafficDirectorManager( |
||||
self, |
||||
) -> td_gamma.TrafficDirectorGammaManager: |
||||
return td_gamma.TrafficDirectorGammaManager( |
||||
self.gcp_api_manager, |
||||
project=self.project, |
||||
resource_prefix=self.resource_prefix, |
||||
resource_suffix=self.resource_suffix, |
||||
network=self.network, |
||||
compute_api_version=self.compute_api_version, |
||||
) |
||||
|
||||
def initKubernetesServerRunner(self) -> GammaServerRunner: |
||||
return GammaServerRunner( |
||||
k8s.KubernetesNamespace( |
||||
self.k8s_api_manager, self.server_namespace |
||||
), |
||||
mesh_name=self.mesh_name, |
||||
server_xds_host=self.server_xds_host, |
||||
deployment_name=self.server_name, |
||||
image_name=self.server_image, |
||||
td_bootstrap_image=self.td_bootstrap_image, |
||||
gcp_project=self.project, |
||||
gcp_api_manager=self.gcp_api_manager, |
||||
gcp_service_account=self.gcp_service_account, |
||||
xds_server_uri=self.xds_server_uri, |
||||
network=self.network, |
||||
debug_use_port_forwarding=self.debug_use_port_forwarding, |
||||
enable_workload_identity=self.enable_workload_identity, |
||||
) |
||||
|
||||
def startTestClient( |
||||
self, test_server: XdsTestServer, **kwargs |
||||
) -> XdsTestClient: |
||||
return super().startTestClient( |
||||
test_server, config_mesh=self.mesh_name_td |
||||
) |
||||
@ -0,0 +1,22 @@ |
||||
--- |
||||
kind: GRPCRoute |
||||
apiVersion: gateway.networking.k8s.io/v1alpha2 |
||||
metadata: |
||||
name: ${route_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
parentRefs: |
||||
- name: ${mesh_name} |
||||
namespace: ${namespace_name} |
||||
group: net.gke.io |
||||
kind: TDMesh |
||||
hostnames: |
||||
- ${xds_server_uri} |
||||
rules: |
||||
- backendRefs: |
||||
- name: ${service_name} |
||||
port: ${test_port} |
||||
namespace: ${namespace_name} |
||||
... |
||||
@ -0,0 +1,17 @@ |
||||
--- |
||||
apiVersion: v1 |
||||
kind: Service |
||||
metadata: |
||||
name: ${service_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
type: ClusterIP |
||||
selector: |
||||
app: ${deployment_name} |
||||
ports: |
||||
- port: ${test_port} |
||||
protocol: TCP |
||||
targetPort: ${test_port} |
||||
... |
||||
@ -0,0 +1,19 @@ |
||||
--- |
||||
kind: TDMesh |
||||
apiVersion: net.gke.io/v1alpha1 |
||||
metadata: |
||||
name: ${mesh_name} |
||||
namespace: ${namespace_name} |
||||
labels: |
||||
owner: xds-k8s-interop-test |
||||
spec: |
||||
gatewayClassName: gke-td |
||||
allowedRoutes: |
||||
namespaces: |
||||
from: All |
||||
kinds: |
||||
- group: net.gke.io |
||||
# This is intentionally incorrect and should be set to GRPCRoute. |
||||
# TODO(sergiitk): [GAMMA] Change when the fix is ready. |
||||
kind: TDGRPCRoute |
||||
... |
||||
@ -0,0 +1,13 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
@ -0,0 +1,44 @@ |
||||
# Copyright 2023 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
import logging |
||||
|
||||
from absl import flags |
||||
from absl.testing import absltest |
||||
|
||||
from framework import xds_gamma_testcase |
||||
from framework import xds_k8s_testcase |
||||
|
||||
logger = logging.getLogger(__name__) |
||||
flags.adopt_module_key_flags(xds_k8s_testcase) |
||||
|
||||
_XdsTestServer = xds_k8s_testcase.XdsTestServer |
||||
_XdsTestClient = xds_k8s_testcase.XdsTestClient |
||||
|
||||
|
||||
class GammaBaselineTest(xds_gamma_testcase.GammaXdsKubernetesTestCase): |
||||
def test_ping_pong(self): |
||||
# TODO(sergiitk): [GAMMA] Consider moving out custom gamma |
||||
# resource creation out of self.startTestServers() |
||||
with self.subTest("1_run_test_server"): |
||||
test_server: _XdsTestServer = self.startTestServers()[0] |
||||
|
||||
with self.subTest("2_start_test_client"): |
||||
test_client: _XdsTestClient = self.startTestClient(test_server) |
||||
|
||||
with self.subTest("3_test_server_received_rpcs_from_test_client"): |
||||
self.assertSuccessfulRpcs(test_client) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
absltest.main() |
||||
Loading…
Reference in new issue