From 198a9f6fe97d45823abb22cee6f3395db086a957 Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Fri, 10 Mar 2023 14:15:01 -0500 Subject: [PATCH] PSM Interop: Local dev various improvements (#32575) PSM Interop: Local dev various improvements - Cleanup resources on ctrl+c - Add startup probes to address the issue with port forwarding starting before the workload listens on a port - Remove misleading restartPolicy: it's silently ignored by k8s - Extra debug message with port-forwarding command --- .../k8s_internal/k8s_port_forwarder.py | 2 ++ .../framework/xds_k8s_testcase.py | 30 +++++++++++++++++-- .../client-secure.deployment.yaml | 5 +++- .../client.deployment.yaml | 5 +++- .../server-secure.deployment.yaml | 5 +++- .../server.deployment.yaml | 5 +++- 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py index 54458f21d4a..a2c9e684246 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py @@ -52,6 +52,8 @@ class PortForwarder: "port-forward", "--address", self.local_address, self.destination, port_mapping ] + logger.debug('Executing port forwarding subprocess cmd: %s', + ' '.join(cmd)) self.subprocess = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py index f8d3ed271cc..f741a93d582 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py @@ -18,8 +18,10 @@ import enum import hashlib import logging import re +import signal import time -from typing import List, Optional, Tuple +from types import FrameType +from typing import Any, Callable, List, Optional, Tuple, Union from absl import flags from absl.testing import absltest @@ -65,6 +67,9 @@ LoadBalancerStatsResponse = grpc_testing.LoadBalancerStatsResponse _ChannelState = grpc_channelz.ChannelState _timedelta = datetime.timedelta ClientConfig = grpc_csds.ClientConfig +# pylint complains about signal.Signals for some reason. +_SignalNum = Union[int, signal.Signals] # pylint: disable=no-member +_SignalHandler = Callable[[_SignalNum, Optional[FrameType]], Any] _TD_CONFIG_MAX_WAIT_SEC = 600 @@ -97,6 +102,8 @@ class XdsKubernetesBaseTestCase(absltest.TestCase): server_xds_port: int td: TrafficDirectorManager td_bootstrap_image: str + _prev_sigint_handler: Optional[_SignalHandler] = None + _handling_sigint: bool = False @staticmethod def is_supported(config: skips.TestConfig) -> bool: @@ -173,13 +180,32 @@ class XdsKubernetesBaseTestCase(absltest.TestCase): cls.secondary_k8s_api_manager.close() cls.gcp_api_manager.close() + def setUp(self): + self._prev_sigint_handler = signal.signal(signal.SIGINT, + self.handle_sigint) + + def handle_sigint(self, signalnum: _SignalNum, + frame: Optional[FrameType]) -> None: + logger.info('Caught Ctrl+C, cleaning up...') + self._handling_sigint = True + # Force resource cleanup by their name. Addresses the case where ctrl-c + # is pressed while waiting for the resource creation. + self.force_cleanup = True + self.tearDown() + self.tearDownClass() + self._handling_sigint = False + if self._prev_sigint_handler is not None: + signal.signal(signal.SIGINT, self._prev_sigint_handler) + raise KeyboardInterrupt + @contextlib.contextmanager def subTest(self, msg, **params): # noqa pylint: disable=signature-differs logger.info('--- Starting subTest %s.%s ---', self.id(), msg) try: yield super().subTest(msg, **params) finally: - logger.info('--- Finished subTest %s.%s ---', self.id(), msg) + if not self._handling_sigint: + logger.info('--- Finished subTest %s.%s ---', self.id(), msg) def setupTrafficDirectorGrpc(self): self.td.setup_for_grpc(self.server_xds_host, diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml index b2afd0e9d1f..b0b76617705 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml @@ -28,7 +28,10 @@ spec: - name: ${deployment_name} image: ${image_name} imagePullPolicy: Always - restartPolicy: Never + startupProbe: + tcpSocket: + port: ${stats_port} + periodSeconds: 3 args: - "--server=${server_target}" - "--stats_port=${stats_port}" diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml index 771365eeeb0..6fd4f0d495f 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml @@ -28,7 +28,10 @@ spec: - name: ${deployment_name} image: ${image_name} imagePullPolicy: Always - restartPolicy: Never + startupProbe: + tcpSocket: + port: ${stats_port} + periodSeconds: 3 args: - "--server=${server_target}" - "--stats_port=${stats_port}" diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml index fd3c7ec2f8f..0727f7e0c31 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml @@ -28,7 +28,10 @@ spec: - name: ${deployment_name} image: ${image_name} imagePullPolicy: Always - restartPolicy: Never + startupProbe: + tcpSocket: + port: ${maintenance_port} + periodSeconds: 3 args: - "--port=${test_port}" - "--maintenance_port=${maintenance_port}" diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml index 7898592bddd..c81d1e6bfa0 100644 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml +++ b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml @@ -28,7 +28,10 @@ spec: - name: ${deployment_name} image: ${image_name} imagePullPolicy: Always - restartPolicy: Never + startupProbe: + tcpSocket: + port: ${test_port} + periodSeconds: 3 args: - "--port=${test_port}" ports: