From 5fbc1a841b0ec810e893f82d6310d115dae64a11 Mon Sep 17 00:00:00 2001 From: Xuan Wang Date: Thu, 17 Aug 2023 15:00:39 -0700 Subject: [PATCH] [PSM Interop] Print stacktrace before test case teardown (#34023) * Before change, stacktrace will only be printed at the end of test, after change, we'll also print the stacktrace before test teardown. * Stacktrace format is similar to [unittest/runner.py](https://github.com/python/cpython/blob/3.10/Lib/unittest/runner.py#L112,L125) * Sample log for url_map (from [this test run](https://source.cloud.google.com/results/invocations/2345d431-6202-478b-97c2-2a1b64bd13c0)): ``` [ FAILED ] csds_test.TestBasicCsds.test_client_config I0810 18:36:25.332329 139712423387136 base_testcase.py:34] ----- TestCase csds_test.TestBasicCsds.test_client_config FAILED ----- E0810 18:36:25.332529 139712423387136 base_testcase.py:53] ERROR Traceback in: csds_test.TestBasicCsds.test_client_config: E0810 18:36:25.332598 139712423387136 base_testcase.py:54] Traceback (most recent call last): File "/tmp/tmp.qFY6iMjIkq/grpc/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py", line 492, in test_client_config retryer(self._fetch_and_check_xds_config) File "/tmp/tmp.qFY6iMjIkq/grpc/tools/run_tests/xds_k8s_test_driver/venv/lib/python3.10/site-packages/tenacity/__init__.py", line 423, in __call__ do = self.iter(retry_state=retry_state) File "/tmp/tmp.qFY6iMjIkq/grpc/tools/run_tests/xds_k8s_test_driver/venv/lib/python3.10/site-packages/tenacity/__init__.py", line 369, in iter return self.retry_error_callback(retry_state=retry_state) File "/tmp/tmp.qFY6iMjIkq/grpc/tools/run_tests/xds_k8s_test_driver/framework/helpers/retryers.py", line 141, in error_handler raise RetryError( framework.helpers.retryers.RetryError: Retry error calling framework.xds_url_map_testcase.XdsUrlMapTestCase._fetch_and_check_xds_config: timeout 0:10:00 (h:mm:ss) exceeded. Last exception: AssertionError: 2 != 3 I0810 18:36:25.332715 139712423387136 xds_url_map_testcase.py:478] Aborting TestBasicCsds I0810 18:36:25.332844 139712423387136 xds_url_map_testcase.py:408] ----- TestCase TestBasicCsds teardown ----- ``` * Sample log when testing locally with multiple errors: ``` Running tests under Python 3.10.9: /usr/local/google/home/xuanwn/.pyenv/versions/310xds/bin/python I0814 17:07:22.553086 140334944188224 xds_k8s_testcase.py:130] ----- Testing BaselineTest ----- I0814 17:07:22.553292 140334944188224 xds_k8s_testcase.py:131] Logs timezone: UTC I0814 17:07:22.553575 140334944188224 skips.py:124] Detected language and version: TestConfig(client_lang='java', server_lang='java', version=None) I0814 17:07:22.592908 140334944188224 k8s.py:129] Using kubernetes context "gke_xuanwn-xds_us-central1-a_xds-k8s-interop-tests-cluster", active host: https://34.132.21.170 I0814 17:07:22.733086 140334944188224 k8s.py:129] Using kubernetes context "None", active host: https://34.144.123.58 [ RUN ] BaselineTest.test_another I0814 17:07:22.740435 140334944188224 xds_k8s_testcase.py:590] Test run resource prefix: xds-k8s-test, suffix: dev I0814 17:07:29.515993 140334944188224 xds_k8s_testcase.py:640] ----- TestMethod __main__.BaselineTest.test_another teardown ----- I0814 17:07:29.517023 140334944188224 xds_k8s_testcase.py:664] ----- Test client/server logs ----- I0814 17:07:29.517564 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-client I0814 17:07:29.517683 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-server [ FAILED ] BaselineTest.test_another I0814 17:07:29.518238 140334944188224 base_testcase.py:40] ----- TestCase __main__.BaselineTest.test_another FAILED ----- E0814 17:07:29.518383 140334944188224 base_testcase.py:61] FAILURE Traceback in __main__.BaselineTest.test_another: Traceback (most recent call last): File "/usr/local/google/home/xuanwn/workspace/xds/grpc/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py", line 34, in test_another self.assertEqual("test another", None) AssertionError: 'test another' != None [ RUN ] BaselineTest.test_another_2 I0814 17:07:29.518589 140334944188224 xds_k8s_testcase.py:590] Test run resource prefix: xds-k8s-test, suffix: dev I0814 17:07:29.535759 140334944188224 xds_k8s_testcase.py:640] ----- TestMethod __main__.BaselineTest.test_another_2 teardown ----- I0814 17:07:29.536597 140334944188224 xds_k8s_testcase.py:664] ----- Test client/server logs ----- I0814 17:07:29.536783 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-client I0814 17:07:29.536932 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-server [ OK ] BaselineTest.test_another_2 I0814 17:07:29.537171 140334944188224 base_testcase.py:53] ----- TestCase __main__.BaselineTest.test_another_2 PASSED ----- [ RUN ] BaselineTest.test_expect_error I0814 17:07:29.537445 140334944188224 xds_k8s_testcase.py:590] Test run resource prefix: xds-k8s-test, suffix: dev I0814 17:07:29.554539 140334944188224 xds_k8s_testcase.py:640] ----- TestMethod __main__.BaselineTest.test_expect_error teardown ----- I0814 17:07:29.555363 140334944188224 xds_k8s_testcase.py:664] ----- Test client/server logs ----- I0814 17:07:29.555552 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-client I0814 17:07:29.555636 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-server [ FAILED ] BaselineTest.test_expect_error I0814 17:07:29.555815 140334944188224 base_testcase.py:46] ----- TestCase __main__.BaselineTest.test_expect_error UNEXPECTEDLY SUCCEED ----- [ RUN ] BaselineTest.test_skip [ SKIPPED ] BaselineTest.test_skip I0814 17:07:29.555988 140334944188224 base_testcase.py:50] ----- TestCase __main__.BaselineTest.test_skip SKIPPED ----- I0814 17:07:29.556062 140334944188224 base_testcase.py:51] Reason for skipping: ['skip for once'] [ RUN ] BaselineTest.test_traffic_director_grpc_setup I0814 17:07:29.556408 140334944188224 xds_k8s_testcase.py:590] Test run resource prefix: xds-k8s-test, suffix: dev I0814 17:07:29.572924 140334944188224 xds_k8s_testcase.py:640] ----- TestMethod __main__.BaselineTest.test_traffic_director_grpc_setup teardown ----- I0814 17:07:29.573563 140334944188224 xds_k8s_testcase.py:664] ----- Test client/server logs ----- I0814 17:07:29.573725 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-client I0814 17:07:29.573856 140334944188224 k8s_base_runner.py:621] No completed deployments of psm-grpc-server [ FAILED ] BaselineTest.test_traffic_director_grpc_setup I0814 17:07:29.574180 140334944188224 base_testcase.py:40] ----- TestCase __main__.BaselineTest.test_traffic_director_grpc_setup FAILED ----- E0814 17:07:29.574276 140334944188224 base_testcase.py:61] FAILURE Traceback in __main__.BaselineTest.test_traffic_director_grpc_setup: Traceback (most recent call last): File "/usr/local/google/home/xuanwn/workspace/xds/grpc/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py", line 31, in test_traffic_director_grpc_setup self.assertEqual("test grpc setup", None) AssertionError: 'test grpc setup' != None ====================================================================== FAIL: test_another (__main__.BaselineTest) BaselineTest.test_another ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/google/home/xuanwn/workspace/xds/grpc/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py", line 34, in test_another self.assertEqual("test another", None) AssertionError: 'test another' != None ====================================================================== FAIL: test_traffic_director_grpc_setup (__main__.BaselineTest) BaselineTest.test_traffic_director_grpc_setup ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/google/home/xuanwn/workspace/xds/grpc/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py", line 31, in test_traffic_director_grpc_setup self.assertEqual("test grpc setup", None) AssertionError: 'test grpc setup' != None ---------------------------------------------------------------------- Ran 5 tests in 7.021s FAILED (failures=2, skipped=1, unexpected successes=1) ``` --- .../framework/test_cases/__init__.py | 13 ++++ .../framework/test_cases/base_testcase.py | 65 +++++++++++++++++++ .../framework/xds_k8s_testcase.py | 4 +- .../framework/xds_url_map_testcase.py | 6 +- 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py create mode 100644 tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py new file mode 100644 index 00000000000..d921d237a32 --- /dev/null +++ b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py new file mode 100644 index 00000000000..d2990323223 --- /dev/null +++ b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py @@ -0,0 +1,65 @@ +# Copyright 2023 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Base test case used for xds test suites.""" + +from typing import Optional +import unittest + +from absl import logging +from absl.testing import absltest + + +class BaseTestCase(absltest.TestCase): + def run(self, result: Optional[unittest.TestResult] = None) -> None: + super().run(result) + test_errors = [error for test, error in result.errors if test is self] + test_failures = [ + failure for test, failure in result.failures if test is self + ] + test_unexpected_successes = [ + test for test in result.unexpectedSuccesses if test is self + ] + test_skipped = next( + (reason for test, reason in result.skipped if test is self), + None, + ) + # Assume one test case will only have one status. + if test_errors or test_failures: + logging.info("----- TestCase %s FAILED -----", self.id()) + if test_errors: + self._print_error_list(test_errors, is_unexpected_error=True) + if test_failures: + self._print_error_list(test_failures) + elif test_unexpected_successes: + logging.info( + "----- TestCase %s UNEXPECTEDLY SUCCEEDED -----", self.id() + ) + elif test_skipped: + logging.info("----- TestCase %s SKIPPED -----", self.id()) + logging.info("Reason for skipping: %s", test_skipped) + else: + logging.info("----- TestCase %s PASSED -----", self.id()) + + def _print_error_list( + self, errors: list[str], is_unexpected_error: bool = False + ) -> None: + # FAILUREs are those errors explicitly signalled using + # the TestCase.assert*() methods. + for err in errors: + logging.error( + "%s Traceback in %s:\n%s", + "ERROR" if is_unexpected_error else "FAILURE", + self.id(), + err, + ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py index bae651a6423..8d7752e6bb2 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py @@ -24,7 +24,6 @@ from types import FrameType from typing import Any, Callable, List, Optional, Tuple, Union from absl import flags -from absl.testing import absltest from google.protobuf import json_format import grpc @@ -46,6 +45,7 @@ from framework.test_app import client_app from framework.test_app import server_app from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner +from framework.test_cases import base_testcase logger = logging.getLogger(__name__) # TODO(yashkt): We will no longer need this flag once Core exposes local certs @@ -84,7 +84,7 @@ class TdPropagationRetryableError(Exception): """Indicates that TD config hasn't propagated yet, and it's safe to retry""" -class XdsKubernetesBaseTestCase(absltest.TestCase): +class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase): lang_spec: skips.TestConfig client_namespace: str client_runner: KubernetesClientRunner diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py index 6fa1c1d5884..c2716aaccad 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py @@ -26,7 +26,6 @@ import unittest from absl import flags from absl import logging -from absl.testing import absltest from google.protobuf import json_format import grpc @@ -38,6 +37,7 @@ from framework.helpers import skips from framework.infrastructure import k8s from framework.test_app import client_app from framework.test_app.runners.k8s import k8s_xds_client_runner +from framework.test_cases import base_testcase # Load existing flags flags.adopt_module_key_flags(xds_k8s_testcase) @@ -268,7 +268,9 @@ class _MetaXdsUrlMapTestCase(type): return new_class -class XdsUrlMapTestCase(absltest.TestCase, metaclass=_MetaXdsUrlMapTestCase): +class XdsUrlMapTestCase( + base_testcase.BaseTestCase, metaclass=_MetaXdsUrlMapTestCase +): """XdsUrlMapTestCase is the base class for urlMap related tests. The subclass is expected to implement 3 methods: