xds-k8s tests: Use test driver from master branch (#27462

An error occurred

)

Instead of directly sourcing the test driver provisioning script from the same branch, the script is downloaded (with curl) and sourced from the master branch. This allows changes made to the test driver to be reflected in all future release branches. A separate PR will backport this change to existing release branches. All cluster definitions are also moved to the install script, allowing any cluster changes to be done in one place in the master branch.
4 years ago · b048618f11
parent cb8dafa248
commit b048618f11
8 changed files with 135 additions and 47 deletions
--- a/tools/internal_ci/linux/grpc_xds_k8s.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s.sh
@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex -o igncr || set -ex
+set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
@ -136,8 +134,13 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
--- a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
@ -24,6 +24,72 @@ readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}"
 readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver"
 readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing"

+# GKE cluster identifiers.
+readonly GKE_CLUSTER_PSM_LB="psm-lb"
+readonly GKE_CLUSTER_PSM_SECURITY="psm-security"
+readonly GKE_CLUSTER_PSM_BASIC="psm-basic"
+
+#######################################
+# Determines the cluster name and zone based on the given cluster identifier.
+# Globals:
+#   GKE_CLUSTER_NAME: Set to reflect the cluster name to use
+#   GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use.
+# Arguments:
+#   The cluster identifier
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+activate_gke_cluster() {
+  case $1 in
+    GKE_CLUSTER_PSM_LB)
+      GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-central1-a"
+      GKE_CLUSTER_ZONE="us-central1-a"
+      ;;
+    GKE_CLUSTER_PSM_SECURITY)
+      GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
+      GKE_CLUSTER_ZONE="us-central1-a"
+      ;;
+    GKE_CLUSTER_PSM_BASIC)
+      GKE_CLUSTER_NAME="interop-test-psm-basic"
+      GKE_CLUSTER_ZONE="us-central1-c"
+      ;;      
+    *)
+      echo "Unknown GKE cluster: ${1}"
+      exit 1
+      ;;
+  esac
+  echo "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}"
+}
+
+#######################################
+# Determines the secondary cluster name and zone based on the given cluster
+# identifier.
+# Globals:
+#   GKE_CLUSTER_NAME: Set to reflect the cluster name to use
+#   GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use.
+# Arguments:
+#   The cluster identifier
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+activate_secondary_gke_cluster() {
+  case $1 in
+    GKE_CLUSTER_PSM_LB)
+      SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-west1-b"
+      SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+      ;;
+    GKE_CLUSTER_PSM_SECURITY)
+      SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-west1-b"
+      SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+      ;;
+    *)
+      echo "Unknown secondary GKE cluster: ${1}"
+      exit 1
+      ;;
+  esac
+  echo "Activated secondary GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}"
+}
+
 #######################################
 # Run command end report its exit code. Doesn't exit on non-zero exit code.
 # Globals:
@ -342,13 +408,12 @@ kokoro_setup_test_driver() {
 #   TEST_DRIVER_REPO_DIR: Unless provided, populated with a temporary dir with
 #                         the path to the test driver repo
 #   SRC_DIR: Populated with absolute path to the source repo
-#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
 #   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
 #   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
 #   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
 #   GIT_COMMIT: Populated with the SHA-1 of git commit being built
-#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
-#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built  
 #   SECONDARY_KUBE_CONTEXT: Populated with name of kubectl context with secondary GKE cluster access, if any
 # Arguments:
 #   The path to the folder containing the build script
@ -358,12 +423,14 @@ kokoro_setup_test_driver() {
 local_setup_test_driver() {
  local script_dir="${1:?Usage: local_setup_test_driver SCRIPT_DIR}"
  readonly SRC_DIR="$(git -C "${script_dir}" rev-parse --show-toplevel)"
-  parse_src_repo_git_info SRC_DIR
+  parse_src_repo_git_info "${SRC_DIR}"
  readonly KUBE_CONTEXT="${KUBE_CONTEXT:-$(kubectl config current-context)}"
  readonly SECONDARY_KUBE_CONTEXT="${SECONDARY_KUBE_CONTEXT}"
+
  local test_driver_repo_dir
  test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}"
  test_driver_install "${test_driver_repo_dir}"
+
  # shellcheck disable=SC2034  # Used in the main script
  readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg"
  # Test out
--- a/tools/internal_ci/linux/grpc_xds_k8s_lb.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_lb.sh
@ -17,11 +17,7 @@ set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-readonly SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-lb-v1-us-west1-b"
-readonly SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
@ -139,8 +135,14 @@ run_alpha_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_LB
+  activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
@ -148,6 +150,7 @@ main() {
    local_setup_test_driver "${script_dir}"
  fi
  build_docker_images_if_needed
+
  # Run tests
  cd "${TEST_DRIVER_FULL_DIR}"
  local failed_tests=0
--- a/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh
@ -17,11 +17,7 @@ set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-readonly SECONDARY_GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-west1-b"
-readonly SECONDARY_GKE_CLUSTER_ZONE="us-west1-b"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server:13171a8b293837517c0446ec0e149e9d10ea3d10"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
@ -133,8 +129,14 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+  
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+  activate_secondary_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
@ -142,6 +144,7 @@ main() {
    local_setup_test_driver "${script_dir}"
  fi
  build_docker_images_if_needed
+
  # Run tests
  cd "${TEST_DRIVER_FULL_DIR}"
  local failed_tests=0
--- a/tools/internal_ci/linux/grpc_xds_k8s_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_python.sh
@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex -o igncr || set -ex
+set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-server"
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
@ -151,8 +149,13 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
--- a/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_xlang.sh
@ -17,10 +17,7 @@ set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
-readonly GKE_CLUSTER_ZONE="us-central1-a"
-
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test server/client Docker images
 readonly IMAGE_REPO="gcr.io/grpc-testing/xds-interop"
 readonly SERVER_LANG="cpp go java"
@ -85,8 +82,14 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Clone the test driver from the master branch using an external script.
+  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_clone_driver_repo.sh
+  source "${script_dir}/grpc_xds_k8s_clone_driver_repo.sh"
+  clone_test_driver
+
+  activate_gke_cluster GKE_CLUSTER_PSM_SECURITY
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
--- a/tools/internal_ci/linux/grpc_xds_url_map.sh
+++ b/tools/internal_ci/linux/grpc_xds_url_map.sh
@ -13,13 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -ex -o igncr || set -ex
+set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-basic"
-readonly GKE_CLUSTER_ZONE="us-central1-c"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
 readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
@ -120,8 +118,13 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_BASIC
+
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
--- a/tools/internal_ci/linux/grpc_xds_url_map_python.sh
+++ b/tools/internal_ci/linux/grpc_xds_url_map_python.sh
@ -17,9 +17,7 @@ set -eo pipefail

 # Constants
 readonly GITHUB_REPOSITORY_NAME="grpc"
-# GKE Cluster
-readonly GKE_CLUSTER_NAME="interop-test-psm-basic"
-readonly GKE_CLUSTER_ZONE="us-central1-c"
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh"
 ## xDS test client Docker images
 readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
 readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
@ -130,8 +128,13 @@ run_test() {
 main() {
  local script_dir
  script_dir="$(dirname "$0")"
-  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
-  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+
+  # Source the test driver from the master branch.
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")"
+
+  activate_gke_cluster GKE_CLUSTER_PSM_BASIC
+  
  set -x
  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"