|
|
|
@ -14,6 +14,7 @@ |
|
|
|
|
""" |
|
|
|
|
Run xDS Test Client on Kubernetes. |
|
|
|
|
""" |
|
|
|
|
import datetime |
|
|
|
|
import logging |
|
|
|
|
from typing import List, Optional |
|
|
|
|
|
|
|
|
@ -30,70 +31,82 @@ class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): |
|
|
|
|
DEFAULT_MAINTENANCE_PORT = 8080 |
|
|
|
|
DEFAULT_SECURE_MODE_MAINTENANCE_PORT = 8081 |
|
|
|
|
|
|
|
|
|
# Required fields. |
|
|
|
|
deployment_template: str |
|
|
|
|
service_name: str |
|
|
|
|
service_template: str |
|
|
|
|
reuse_service: bool |
|
|
|
|
enable_workload_identity: bool |
|
|
|
|
debug_use_port_forwarding: bool |
|
|
|
|
gcp_neg_name: str |
|
|
|
|
td_bootstrap_image: str |
|
|
|
|
xds_server_uri: str |
|
|
|
|
network: str |
|
|
|
|
|
|
|
|
|
# Optional fields. |
|
|
|
|
service_account_name: Optional[str] = None |
|
|
|
|
service_account_template: Optional[str] = None |
|
|
|
|
gcp_iam: Optional[gcp.iam.IamV1] = None |
|
|
|
|
|
|
|
|
|
# Mutable state. |
|
|
|
|
service: Optional[k8s.V1Service] = None |
|
|
|
|
|
|
|
|
|
def __init__( # pylint: disable=too-many-locals |
|
|
|
|
self, |
|
|
|
|
k8s_namespace, |
|
|
|
|
k8s_namespace: k8s.KubernetesNamespace, |
|
|
|
|
*, |
|
|
|
|
deployment_name, |
|
|
|
|
image_name, |
|
|
|
|
td_bootstrap_image, |
|
|
|
|
deployment_name: str, |
|
|
|
|
image_name: str, |
|
|
|
|
td_bootstrap_image: str, |
|
|
|
|
network: str = 'default', |
|
|
|
|
xds_server_uri: Optional[str] = None, |
|
|
|
|
gcp_api_manager: gcp.api.GcpApiManager, |
|
|
|
|
gcp_project: str, |
|
|
|
|
gcp_service_account: str, |
|
|
|
|
service_account_name=None, |
|
|
|
|
service_name=None, |
|
|
|
|
neg_name=None, |
|
|
|
|
xds_server_uri=None, |
|
|
|
|
network='default', |
|
|
|
|
deployment_template='server.deployment.yaml', |
|
|
|
|
service_account_template='service-account.yaml', |
|
|
|
|
service_template='server.service.yaml', |
|
|
|
|
reuse_service=False, |
|
|
|
|
reuse_namespace=False, |
|
|
|
|
namespace_template=None, |
|
|
|
|
debug_use_port_forwarding=False, |
|
|
|
|
enable_workload_identity=True): |
|
|
|
|
super().__init__(k8s_namespace, namespace_template, reuse_namespace) |
|
|
|
|
service_account_name: Optional[str] = None, |
|
|
|
|
service_name: Optional[str] = None, |
|
|
|
|
neg_name: Optional[str] = None, |
|
|
|
|
deployment_template: str = 'server.deployment.yaml', |
|
|
|
|
service_account_template: str = 'service-account.yaml', |
|
|
|
|
service_template: str = 'server.service.yaml', |
|
|
|
|
reuse_service: bool = False, |
|
|
|
|
reuse_namespace: bool = False, |
|
|
|
|
namespace_template: Optional[str] = None, |
|
|
|
|
debug_use_port_forwarding: bool = False, |
|
|
|
|
enable_workload_identity: bool = True): |
|
|
|
|
super().__init__(k8s_namespace, |
|
|
|
|
deployment_name=deployment_name, |
|
|
|
|
image_name=image_name, |
|
|
|
|
gcp_project=gcp_project, |
|
|
|
|
gcp_service_account=gcp_service_account, |
|
|
|
|
gcp_ui_url=gcp_api_manager.gcp_ui_url, |
|
|
|
|
namespace_template=namespace_template, |
|
|
|
|
reuse_namespace=reuse_namespace) |
|
|
|
|
|
|
|
|
|
# Settings |
|
|
|
|
self.deployment_name = deployment_name |
|
|
|
|
self.image_name = image_name |
|
|
|
|
self.service_name = service_name or deployment_name |
|
|
|
|
# xDS bootstrap generator |
|
|
|
|
self.td_bootstrap_image = td_bootstrap_image |
|
|
|
|
self.xds_server_uri = xds_server_uri |
|
|
|
|
# This only works in k8s >= 1.18.10-gke.600 |
|
|
|
|
# https://cloud.google.com/kubernetes-engine/docs/how-to/standalone-neg#naming_negs |
|
|
|
|
self.neg_name = neg_name or (f'{self.k8s_namespace.name}-' |
|
|
|
|
f'{self.service_name}') |
|
|
|
|
self.network = network |
|
|
|
|
self.deployment_template = deployment_template |
|
|
|
|
self.service_name = service_name or deployment_name |
|
|
|
|
self.service_template = service_template |
|
|
|
|
self.reuse_service = reuse_service |
|
|
|
|
self.debug_use_port_forwarding = debug_use_port_forwarding |
|
|
|
|
self.enable_workload_identity = enable_workload_identity |
|
|
|
|
# Service account settings: |
|
|
|
|
# Kubernetes service account |
|
|
|
|
self.debug_use_port_forwarding = debug_use_port_forwarding |
|
|
|
|
# GCP Network Endpoint Group. |
|
|
|
|
self.gcp_neg_name = neg_name or (f'{self.k8s_namespace.name}-' |
|
|
|
|
f'{self.service_name}') |
|
|
|
|
|
|
|
|
|
# Used by the TD bootstrap generator. |
|
|
|
|
self.td_bootstrap_image = td_bootstrap_image |
|
|
|
|
self.network = network |
|
|
|
|
self.xds_server_uri = xds_server_uri |
|
|
|
|
|
|
|
|
|
# Workload identity settings: |
|
|
|
|
if self.enable_workload_identity: |
|
|
|
|
# Kubernetes service account. |
|
|
|
|
self.service_account_name = service_account_name or deployment_name |
|
|
|
|
self.service_account_template = service_account_template |
|
|
|
|
else: |
|
|
|
|
self.service_account_name = None |
|
|
|
|
self.service_account_template = None |
|
|
|
|
|
|
|
|
|
# GCP. |
|
|
|
|
self.gcp_project = gcp_project |
|
|
|
|
self.gcp_ui_url = gcp_api_manager.gcp_ui_url |
|
|
|
|
# GCP service account to map to Kubernetes service account |
|
|
|
|
self.gcp_service_account = gcp_service_account |
|
|
|
|
# GCP IAM API used to grant allow workload service accounts permission |
|
|
|
|
# to use GCP service account identity. |
|
|
|
|
self.gcp_iam = gcp.iam.IamV1(gcp_api_manager, gcp_project) |
|
|
|
|
|
|
|
|
|
# Mutable state |
|
|
|
|
self.deployment: Optional[k8s.V1Deployment] = None |
|
|
|
|
self.service_account: Optional[k8s.V1ServiceAccount] = None |
|
|
|
|
self.service: Optional[k8s.V1Service] = None |
|
|
|
|
# GCP IAM API used to grant allow workload service accounts |
|
|
|
|
# permission to use GCP service account identity. |
|
|
|
|
self.gcp_iam = gcp.iam.IamV1(gcp_api_manager, gcp_project) |
|
|
|
|
|
|
|
|
|
def run( # pylint: disable=arguments-differ,too-many-branches |
|
|
|
|
self, |
|
|
|
@ -126,12 +139,6 @@ class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): |
|
|
|
|
'maintenance_port=%s secure_mode=%s replica_count=%s', |
|
|
|
|
self.deployment_name, self.k8s_namespace.name, test_port, |
|
|
|
|
maintenance_port, secure_mode, replica_count) |
|
|
|
|
self._logs_explorer_link(deployment_name=self.deployment_name, |
|
|
|
|
namespace_name=self.k8s_namespace.name, |
|
|
|
|
gcp_project=self.gcp_project, |
|
|
|
|
gcp_ui_url=self.gcp_ui_url) |
|
|
|
|
|
|
|
|
|
# Create namespace. |
|
|
|
|
super().run() |
|
|
|
|
|
|
|
|
|
# Reuse existing if requested, create a new deployment when missing. |
|
|
|
@ -144,7 +151,7 @@ class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): |
|
|
|
|
service_name=self.service_name, |
|
|
|
|
namespace_name=self.k8s_namespace.name, |
|
|
|
|
deployment_name=self.deployment_name, |
|
|
|
|
neg_name=self.neg_name, |
|
|
|
|
neg_name=self.gcp_neg_name, |
|
|
|
|
test_port=test_port) |
|
|
|
|
self._wait_service_neg(self.service_name, test_port) |
|
|
|
|
|
|
|
|
@ -190,6 +197,8 @@ class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): |
|
|
|
|
# Verify the deployment reports all pods started as well. |
|
|
|
|
self._wait_deployment_with_available_replicas(self.deployment_name, |
|
|
|
|
replica_count) |
|
|
|
|
self.time_start_completed = datetime.datetime.now() |
|
|
|
|
|
|
|
|
|
servers: List[XdsTestServer] = [] |
|
|
|
|
for pod in pods: |
|
|
|
|
servers.append( |
|
|
|
@ -228,21 +237,28 @@ class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): |
|
|
|
|
secure_mode=secure_mode, |
|
|
|
|
rpc_host=rpc_host) |
|
|
|
|
|
|
|
|
|
def cleanup(self, *, force=False, force_namespace=False): # pylint: disable=arguments-differ |
|
|
|
|
if self.deployment or force: |
|
|
|
|
self._delete_deployment(self.deployment_name) |
|
|
|
|
self.deployment = None |
|
|
|
|
if (self.service and not self.reuse_service) or force: |
|
|
|
|
self._delete_service(self.service_name) |
|
|
|
|
self.service = None |
|
|
|
|
if self.enable_workload_identity and (self.service_account or force): |
|
|
|
|
self._revoke_workload_identity_user( |
|
|
|
|
gcp_iam=self.gcp_iam, |
|
|
|
|
gcp_service_account=self.gcp_service_account, |
|
|
|
|
service_account_name=self.service_account_name) |
|
|
|
|
self._delete_service_account(self.service_account_name) |
|
|
|
|
self.service_account = None |
|
|
|
|
super().cleanup(force=(force_namespace and force)) |
|
|
|
|
# pylint: disable=arguments-differ |
|
|
|
|
def cleanup(self, *, force=False, force_namespace=False): |
|
|
|
|
try: |
|
|
|
|
if self.deployment or force: |
|
|
|
|
self._delete_deployment(self.deployment_name) |
|
|
|
|
self.deployment = None |
|
|
|
|
if (self.service and not self.reuse_service) or force: |
|
|
|
|
self._delete_service(self.service_name) |
|
|
|
|
self.service = None |
|
|
|
|
if (self.enable_workload_identity and |
|
|
|
|
(self.service_account or force)): |
|
|
|
|
self._revoke_workload_identity_user( |
|
|
|
|
gcp_iam=self.gcp_iam, |
|
|
|
|
gcp_service_account=self.gcp_service_account, |
|
|
|
|
service_account_name=self.service_account_name) |
|
|
|
|
self._delete_service_account(self.service_account_name) |
|
|
|
|
self.service_account = None |
|
|
|
|
self._cleanup_namespace(force=(force_namespace and force)) |
|
|
|
|
finally: |
|
|
|
|
self.time_stopped = datetime.datetime.now() |
|
|
|
|
|
|
|
|
|
# pylint: enable=arguments-differ |
|
|
|
|
|
|
|
|
|
@classmethod |
|
|
|
|
def make_namespace_name(cls, |
|
|
|
|