From b048618f11344d1f25d485c44d15a56c528eccd7 Mon Sep 17 00:00:00 2001
From: Terry Wilson <terrymwilson@gmail.com>
Date: Thu, 7 Oct 2021 10:47:55 -0700
Subject: [PATCH] xds-k8s tests: Use test driver from master branch (#27462)

Instead of directly sourcing the test driver provisioning script from the same branch, the script is downloaded (with curl) and sourced from the master branch.

This allows changes made to the test driver to be reflected in all future release branches. A separate PR will backport this change to existing release branches.

All cluster definitions are also moved to the install script, allowing any cluster changes to be done in one place in the master branch.
---
 tools/internal_ci/linux/grpc_xds_k8s.sh       | 15 ++--
 .../linux/grpc_xds_k8s_install_test_driver.sh | 75 ++++++++++++++++++-
 tools/internal_ci/linux/grpc_xds_k8s_lb.sh    | 17 +++--
 .../linux/grpc_xds_k8s_lb_python.sh           | 17 +++--
 .../internal_ci/linux/grpc_xds_k8s_python.sh  | 15 ++--
 tools/internal_ci/linux/grpc_xds_k8s_xlang.sh | 15 ++--
 tools/internal_ci/linux/grpc_xds_url_map.sh   | 15 ++--
 .../linux/grpc_xds_url_map_python.sh          | 13 ++--
 8 files changed, 135 insertions(+), 47 deletions(-)

diff --git a/tools/internal_ci/linux/grpc_xds_k8s.sh b/tools/internal_ci/linux/grpc_xds_k8s.sh
index ba9e2cddbed..71bcdb70122 100755
--- a/tools/internal_ci/linux/grpc_xds_k8s.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s.sh
@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex -o igncr || set -ex
+set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
@@ -136,8 +134,13 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
diff --git a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
index ba3beaa1d94..bb333c3f4e5 100644
--- a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
@@ -24,6 +24,72 @@ readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}"
 readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver"
 readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing"
 
+# GKE cluster identifiers.
+readonly GKE_CLUSTER_PSM_LB="psm-lb"
+readonly GKE_CLUSTER_PSM_SECURITY="psm-security"
+readonly GKE_CLUSTER_PSM_BASIC="psm-basic"
+
+#######################################
+# Determines the cluster name and zone based on the given cluster identifier.
+# Globals:
+#   GKE_CLUSTER_NAME: Set to reflect the cluster name to use
+#   GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use.
+# Arguments:
+#   The cluster identifier
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+activate_gke_cluster() {
+  case $1 in
+    GKE_CLUSTER_PSM_LB)
+      GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-central1-a"
+      GKE_CLUSTER_ZONE="us-central1-a"
+      ;;
+    GKE_CLUSTER_PSM_SECURITY)
+      GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
+      GKE_CLUSTER_ZONE="us-central1-a"
+      ;;
+    GKE_CLUSTER_PSM_BASIC)
+      GKE_CLUSTER_NAME="interop-test-psm-basic"
+      GKE_CLUSTER_ZONE="us-central1-c"
+      ;;      
+    *)
+      echo "Unknown GKE cluster: ${1}"
+      exit 1
+      ;;
+  esac
+  echo "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}"
+}
+
+#######################################
+# Determines the secondary cluster name and zone based on the given cluster
+# identifier.
+# Globals:
+#   GKE_CLUSTER_NAME: Set to reflect the cluster name to use
+#   GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use.
+# Arguments:
+#   The cluster identifier
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+activate_secondary_gke_cluster() {
+  case $1 in
+    GKE_CLUSTER_PSM_LB)
+      SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-west1-b"
+      SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+      ;;
+    GKE_CLUSTER_PSM_SECURITY)
+      SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-west1-b"
+      SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+      ;;
+    *)
+      echo "Unknown secondary GKE cluster: ${1}"
+      exit 1
+      ;;
+  esac
+  echo "Activated secondary GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}"
+}
+
 #######################################
 # Run command end report its exit code. Doesn't exit on non-zero exit code.
 # Globals:
@@ -342,13 +408,12 @@ kokoro_setup_test_driver() {
 #   TEST_DRIVER_REPO_DIR: Unless provided, populated with a temporary dir with
 #                         the path to the test driver repo
 #   SRC_DIR: Populated with absolute path to the source repo
-#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
 #   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
 #   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
 #   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
 #   GIT_COMMIT: Populated with the SHA-1 of git commit being built
-#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
-#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built  
 #   SECONDARY_KUBE_CONTEXT: Populated with name of kubectl context with secondary GKE cluster access, if any
 # Arguments:
 #   The path to the folder containing the build script
@@ -358,12 +423,14 @@ kokoro_setup_test_driver() {
 local_setup_test_driver() {
   local script_dir="${1:?Usage: local_setup_test_driver SCRIPT_DIR}"
   readonly SRC_DIR="$(git -C "${script_dir}" rev-parse --show-toplevel)"
-  parse_src_repo_git_info SRC_DIR
+  parse_src_repo_git_info "${SRC_DIR}"
   readonly KUBE_CONTEXT="${KUBE_CONTEXT:-$(kubectl config current-context)}"
   readonly SECONDARY_KUBE_CONTEXT="${SECONDARY_KUBE_CONTEXT}"
+
   local test_driver_repo_dir
   test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}"
   test_driver_install "${test_driver_repo_dir}"
+
   # shellcheck disable=SC2034  # Used in the main script
   readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg"
   # Test out
diff --git a/tools/internal_ci/linux/grpc_xds_k8s_lb.sh b/tools/internal_ci/linux/grpc_xds_k8s_lb.sh
index a3595e384a1..606a2541f83 100755
--- a/tools/internal_ci/linux/grpc_xds_k8s_lb.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_lb.sh
@@ -17,11 +17,7 @@ set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-readonly SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-west1-b"
-readonly SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
@@ -139,8 +135,14 @@ run_alpha_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_LB
+  activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
@@ -148,6 +150,7 @@ main() {
     local_setup_test_driver "${script_dir}"
   fi
   build_docker_images_if_needed
+
   # Run tests
   cd "${TEST_DRIVER_FULL_DIR}"
   local failed_tests=0
diff --git a/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh b/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh
index 72766b12d04..7cf7318e72b 100755
--- a/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh
@@ -17,11 +17,7 @@ set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-readonly SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-west1-b"
-readonly SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server:13171a8b293837517c0446ec0e149e9d10ea3d10"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
@@ -133,8 +129,14 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+  
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+  activate_secondary_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
@@ -142,6 +144,7 @@ main() {
     local_setup_test_driver "${script_dir}"
   fi
   build_docker_images_if_needed
+
   # Run tests
   cd "${TEST_DRIVER_FULL_DIR}"
   local failed_tests=0
diff --git a/tools/internal_ci/linux/grpc_xds_k8s_python.sh b/tools/internal_ci/linux/grpc_xds_k8s_python.sh
index 605102751be..7911596bff0 100755
--- a/tools/internal_ci/linux/grpc_xds_k8s_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_python.sh
@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex -o igncr || set -ex
+set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
@@ -151,8 +149,13 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
diff --git a/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh b/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh
index c7564dfdd19..a424818d1ac 100755
--- a/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh
@@ -17,10 +17,7 @@ set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly IMAGE_REPO="gcr.io/grpc-testing/xds-interop"
 readonly SERVER_LANG="cpp go java"
@@ -85,8 +82,14 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Clone the test driver from the master branch using an external script.
+  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_clone_driver_repo.sh
+  source "${script_dir}/grpc_xds_k8s_clone_driver_repo.sh"
+  clone_test_driver
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
diff --git a/tools/internal_ci/linux/grpc_xds_url_map.sh b/tools/internal_ci/linux/grpc_xds_url_map.sh
index f823f6c5433..696ef97d0d0 100755
--- a/tools/internal_ci/linux/grpc_xds_url_map.sh
+++ b/tools/internal_ci/linux/grpc_xds_url_map.sh
@@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex -o igncr || set -ex
+set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-basic"
-readonly GKE_CLUSTER_ZONE="us-central1-c"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
 readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
@@ -120,8 +118,13 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_BASIC
+
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
diff --git a/tools/internal_ci/linux/grpc_xds_url_map_python.sh b/tools/internal_ci/linux/grpc_xds_url_map_python.sh
index bb302c825d7..7316df954f6 100755
--- a/tools/internal_ci/linux/grpc_xds_url_map_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_url_map_python.sh
@@ -17,9 +17,7 @@ set -eo pipefail
 
 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-basic"
-readonly GKE_CLUSTER_ZONE="us-central1-c"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
 readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
@@ -130,8 +128,13 @@ run_test() {
 main() {
   local script_dir
   script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_BASIC
+  
   set -x
   if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
     kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"