diff --git a/tools/internal_ci/helper_scripts/list_leftover_loadtests.sh b/tools/internal_ci/helper_scripts/list_leftover_loadtests.sh new file mode 100644 index 00000000000..572843caca6 --- /dev/null +++ b/tools/internal_ci/helper_scripts/list_leftover_loadtests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +echo "BEGIN Listing leftover tests." + +# Find tests that have running pods and are in Errored state. +kubectl get pods --no-headers -o jsonpath='{range .items[*]}{.metadata.ownerReferences[0].name}{" "}{.status.phase}{"\n"}{end}' \ + | grep Running \ + | cut -f1 -d' ' \ + | sort -u \ + | xargs -r kubectl get loadtest --no-headers -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.state}{" "}{.metadata.annotations.pool}{" "}{.metadata.annotations.scenario}{"\n"}{end}' \ + | grep Errored + +echo "END Listing leftover tests." diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index fa4ebf73175..e24d0678ca4 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -29,8 +29,9 @@ gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod \ --zone us-central1-b --project grpc-testing -# List pods that may be left over from a previous run. -kubectl get pods | grep -v Completed +# List tests that have running pods and are in errored state. +# This is an unexpected condition, and it is logged here for monitoring. +source tools/internal_ci/helper_scripts/list_leftover_loadtests.sh # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" diff --git a/tools/internal_ci/linux/grpc_e2e_performance_v2.sh b/tools/internal_ci/linux/grpc_e2e_performance_v2.sh index 88ec89461d1..1862503ffba 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_v2.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_v2.sh @@ -29,8 +29,9 @@ gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod \ --zone us-central1-b --project grpc-testing -# List pods that may be left over from a previous run. -kubectl get pods | grep -v Completed +# List tests that have running pods and are in errored state. +# This is an unexpected condition, and it is logged here for monitoring. +source tools/internal_ci/helper_scripts/list_leftover_loadtests.sh # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}"