List leftover tests that have running pods and are in Errored state. (#26296)

* List annotations of tests that have running pods and are in Errored state.

* Fix format.

* Use pod owner reference instead of "loadtest" label.

* Delete loadtests that have running pods and are in Errored state.

* Improve jsonpath expressions.

* Add comment.

* List tests but do not delete them.

tools/internal_ci/helper_scripts/list_leftover_loadtests.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+echo "BEGIN Listing leftover tests."
+
+# Find tests that have running pods and are in Errored state.
+kubectl get pods --no-headers -o jsonpath='{range .items[*]}{.metadata.ownerReferences[0].name}{" "}{.status.phase}{"\n"}{end}' \
+    | grep Running \
+    | cut -f1 -d' ' \
+    | sort -u \
+    | xargs -r kubectl get loadtest --no-headers -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.state}{" "}{.metadata.annotations.pool}{" "}{.metadata.annotations.scenario}{"\n"}{end}' \
+    | grep Errored
+
+echo "END Listing leftover tests."

tools/internal_ci/linux/grpc_e2e_performance_gke.sh

@@ -29,8 +29,9 @@ gcloud config set project grpc-testing
 gcloud container clusters get-credentials benchmarks-prod \
     --zone us-central1-b --project grpc-testing
 
-# List pods that may be left over from a previous run.
-kubectl get pods | grep -v Completed
+# List tests that have running pods and are in errored state.
+# This is an unexpected condition, and it is logged here for monitoring.
+source tools/internal_ci/helper_scripts/list_leftover_loadtests.sh
 
 # Set up environment variables.
 LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}"

tools/internal_ci/linux/grpc_e2e_performance_v2.sh

@@ -29,8 +29,9 @@ gcloud config set project grpc-testing
 gcloud container clusters get-credentials benchmarks-prod \
     --zone us-central1-b --project grpc-testing
 
-# List pods that may be left over from a previous run.
-kubectl get pods | grep -v Completed
+# List tests that have running pods and are in errored state.
+# This is an unexpected condition, and it is logged here for monitoring.
+source tools/internal_ci/helper_scripts/list_leftover_loadtests.sh
 
 # Set up environment variables.
 LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}"