Add kokoro job for xds psm security tests (#26033)

* add kokoro job for xds psm security * Fix scripts * Cleanup * Add copyright
4 years ago · 57cb063fb7
parent 0b8cb0227b
commit 57cb063fb7
5 changed files with 606 additions and 0 deletions
--- a/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_client
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_client
@ -0,0 +1,38 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for building //test/cpp/interop:xds_interop_client
+
+FROM phusion/baseimage:master@sha256:65ea10d5f757e5e86272625f8675d437dd83d8db64bdb429e2354d58f5462750
+
+RUN apt-get update -y && \
+        apt-get install -y \
+            build-essential \
+            clang \
+            python3 \
+            python3-dev
+
+WORKDIR /workdir
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN mkdir /artifacts
+
+COPY . .
+RUN tools/bazel build //test/cpp/interop:xds_interop_client
+RUN cp -rL /workdir/bazel-bin/test/cpp/interop/xds_interop_client /artifacts/
+
+FROM phusion/baseimage:master@sha256:65ea10d5f757e5e86272625f8675d437dd83d8db64bdb429e2354d58f5462750
+COPY --from=0 /artifacts ./
+
+ENTRYPOINT ["/xds_interop_client"]
--- a/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_server
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_server
@ -0,0 +1,38 @@
+# Copyright 2021 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for building //test/cpp/interop:xds_interop_client
+
+FROM phusion/baseimage:master@sha256:65ea10d5f757e5e86272625f8675d437dd83d8db64bdb429e2354d58f5462750
+
+RUN apt-get update -y && \
+        apt-get install -y \
+            build-essential \
+            clang \
+            python3 \
+            python3-dev
+
+WORKDIR /workdir
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN mkdir /artifacts
+
+COPY . .
+RUN tools/bazel build //test/cpp/interop:xds_interop_server
+RUN cp -rL /workdir/bazel-bin/test/cpp/interop/xds_interop_server /artifacts/
+
+FROM phusion/baseimage:master@sha256:65ea10d5f757e5e86272625f8675d437dd83d8db64bdb429e2354d58f5462750
+COPY --from=0 /artifacts ./
+
+ENTRYPOINT ["/xds_interop_server"]
--- a/tools/internal_ci/linux/grpc_xds_k8s.cfg
+++ b/tools/internal_ci/linux/grpc_xds_k8s.cfg
@ -0,0 +1,26 @@
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_xds_k8s.sh"
+timeout_mins: 110
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*sponge_log.log"
+    strip_prefix: "artifacts"
+  }
+}
--- a/tools/internal_ci/linux/grpc_xds_k8s.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s.sh
@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex -o igncr || set -ex
+
+# Constants
+readonly GITHUB_REPOSITORY_NAME="grpc"
+# GKE Cluster
+readonly GKE_CLUSTER_NAME="interop-test-psm-sec-testing-api"
+readonly GKE_CLUSTER_ZONE="us-west1-b"
+export CLOUDSDK_API_ENDPOINT_OVERRIDES_CONTAINER="https://test-container.sandbox.googleapis.com/"
+## xDS test server/client Docker images
+readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-server"
+readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/cpp-client"
+readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
+readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing"
+
+#######################################
+# Builds test app Docker images and pushes them to GCR
+# Globals:
+#   BUILD_APP_PATH
+#   SERVER_IMAGE_NAME: Test server Docker image name
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `gcloud builds submit` to stdout, stderr
+#######################################
+build_test_app_docker_images() {
+  echo "Building C++ xDS interop test app Docker images"
+  docker build -f "${SRC_DIR}/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_client" -t "${CLIENT_IMAGE_NAME}" "${SRC_DIR}"
+  docker build -f "${SRC_DIR}/tools/dockerfile/interoptest/grpc_interop_cxx_xds/Dockerfile.xds_server" -t "${SERVER_IMAGE_NAME}" "${SRC_DIR}"
+  docker push "${CLIENT_IMAGE_NAME}"
+  docker push "${SERVER_IMAGE_NAME}"
+}
+
+#######################################
+# Builds test app and its docker images unless they already exist
+# Globals:
+#   SERVER_IMAGE_NAME: Test server Docker image name
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+#   FORCE_IMAGE_BUILD
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+build_docker_images_if_needed() {
+  # Check if images already exist
+  server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")"
+  printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}"
+  echo "${server_tags:-Server image not found}"
+
+  client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")"
+  printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}"
+  echo "${client_tags:-Client image not found}"
+
+  # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1
+  if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then
+    build_test_app_docker_images
+  else
+    echo "Skipping C++ test app build"
+  fi
+}
+
+#######################################
+# Executes the test case
+# Globals:
+#   TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile
+#   KUBE_CONTEXT: The name of kubectl context with GKE cluster access
+#   TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report
+#   SERVER_IMAGE_NAME: Test server Docker image name
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+# Arguments:
+#   Test case name
+# Outputs:
+#   Writes the output of test execution to stdout, stderr
+#   Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml
+#######################################
+run_test() {
+  # Test driver usage:
+  # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
+  local test_name="${1:?Usage: run_test test_name}"
+  set -x
+  python -m "tests.${test_name}" \
+    --flagfile="${TEST_DRIVER_FLAGFILE}" \
+    --kube_context="${KUBE_CONTEXT}" \
+    --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
+    --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
+    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
+    --force_cleanup
+  set +x
+}
+
+#######################################
+# Main function: provision software necessary to execute tests, and run them
+# Globals:
+#   KOKORO_ARTIFACTS_DIR
+#   GITHUB_REPOSITORY_NAME
+#   SRC_DIR: Populated with absolute path to the source repo
+#   TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing
+#                         the test driver
+#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
+#   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
+#   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
+#   GIT_COMMIT: Populated with the SHA-1 of git commit being built
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of test execution to stdout, stderr
+#######################################
+main() {
+  local script_dir
+  script_dir="$(dirname "$0")"
+  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
+  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+  set -x
+  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
+    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
+  else
+    local_setup_test_driver "${script_dir}"
+  fi
+  build_docker_images_if_needed
+  # Run tests
+  cd "${TEST_DRIVER_FULL_DIR}"
+  run_test baseline_test
+  run_test security_test
+}
+
+main "$@"
--- a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
+++ b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
@ -0,0 +1,356 @@
+#!/usr/bin/env bash
+# Copyright 2020 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# TODO(sergiitk): move to grpc/grpc when implementing support of other languages
+set -eo pipefail
+
+# Constants
+readonly PYTHON_VERSION="3.6"
+# Test driver
+readonly TEST_DRIVER_REPO_NAME="grpc"
+readonly TEST_DRIVER_REPO_URL="https://github.com/grpc/grpc.git"
+readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}"
+readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver"
+readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing"
+
+#######################################
+# Run command end report its exit code. Doesn't exit on non-zero exit code.
+# Globals:
+#   None
+# Arguments:
+#   Command to execute
+# Outputs:
+#   Writes the output of given command to stdout, stderr
+#######################################
+run_ignore_exit_code() {
+  local exit_code=-1
+  "$@" || exit_code=$?
+  echo "Exit code: ${exit_code}"
+}
+
+#######################################
+# Parses information about git repository at given path to global variables.
+# Globals:
+#   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
+#   GIT_COMMIT: Populated with the SHA-1 of git commit being built
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
+# Arguments:
+#   Git source dir
+#######################################
+parse_src_repo_git_info() {
+  local src_dir="${SRC_DIR:?SRC_DIR must be set}"
+  readonly GIT_ORIGIN_URL=$(git -C "${src_dir}" remote get-url origin)
+  readonly GIT_COMMIT=$(git -C "${src_dir}" rev-parse HEAD)
+  readonly GIT_COMMIT_SHORT=$(git -C "${src_dir}" rev-parse --short HEAD)
+}
+
+#######################################
+# List GCR image tags matching given tag name.
+# Arguments:
+#   Image name
+#   Tag name
+# Outputs:
+#   Writes the table with the list of found tags to stdout.
+#   If no tags found, the output is an empty string.
+#######################################
+gcloud_gcr_list_image_tags() {
+  gcloud container images list-tags --format="table[box](tags,digest,timestamp.date())" --filter="tags:$2" "$1"
+}
+
+#######################################
+# A helper to execute `gcloud -q components update`.
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `gcloud` command to stdout, stderr
+#######################################
+gcloud_update() {
+  echo "Update gcloud components:"
+  gcloud -q components update
+}
+
+#######################################
+# Create kube context authenticated with GKE cluster, saves context name.
+# to KUBE_CONTEXT
+# Globals:
+#   GKE_CLUSTER_NAME
+#   GKE_CLUSTER_ZONE
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `gcloud` command to stdout, stderr
+#   Writes authorization info $HOME/.kube/config
+#######################################
+gcloud_get_cluster_credentials() {
+  gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}"
+  readonly KUBE_CONTEXT="$(kubectl config current-context)"
+}
+
+#######################################
+# Clone the source code of the test driver to $TEST_DRIVER_REPO_DIR, unless
+# given folder exists.
+# Globals:
+#   TEST_DRIVER_REPO_URL
+#   TEST_DRIVER_BRANCH
+#   TEST_DRIVER_REPO_DIR: path to the repo containing the test driver
+#   TEST_DRIVER_REPO_DIR_USE_EXISTING: set non-empty value to use exiting
+#      clone of the driver repo located at $TEST_DRIVER_REPO_DIR.
+#      Useful for debugging the build script locally.
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `git` command to stdout, stderr
+#   Writes driver source code to $TEST_DRIVER_REPO_DIR
+#######################################
+test_driver_get_source() {
+  if [[ -n "${TEST_DRIVER_REPO_DIR_USE_EXISTING}" && -d "${TEST_DRIVER_REPO_DIR}" ]]; then
+    echo "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}."
+  else
+    echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}"
+    git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}"
+  fi
+}
+
+#######################################
+# Install Python modules from required in $TEST_DRIVER_FULL_DIR/requirements.txt
+# to Python virtual environment. Creates and activates Python venv if necessary.
+# Globals:
+#   TEST_DRIVER_FULL_DIR
+#   PYTHON_VERSION
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `python`, `pip` commands to stdout, stderr
+#   Writes the list of installed modules to stdout
+#######################################
+test_driver_pip_install() {
+  echo "Install python dependencies"
+  cd "${TEST_DRIVER_FULL_DIR}"
+
+  # Create and activate virtual environment unless already using one
+  if [[ -z "${VIRTUAL_ENV}" ]]; then
+    local venv_dir="${TEST_DRIVER_FULL_DIR}/venv"
+    if [[ -d "${venv_dir}" ]]; then
+      echo "Found python virtual environment directory: ${venv_dir}"
+    else
+      echo "Creating python virtual environment: ${venv_dir}"
+      "python${PYTHON_VERSION} -m venv ${venv_dir}"
+    fi
+    # Intentional: No need to check python venv activate script.
+    # shellcheck source=/dev/null
+    source "${venv_dir}/bin/activate"
+  fi
+
+  pip install -r requirements.txt
+  echo "Installed Python packages:"
+  pip list
+}
+
+#######################################
+# Compile proto-files needed for the test driver
+# Globals:
+#   TEST_DRIVER_REPO_DIR
+#   TEST_DRIVER_FULL_DIR
+#   TEST_DRIVER_PROTOS_PATH
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `python -m grpc_tools.protoc` to stdout, stderr
+#   Writes the list if compiled python code to stdout
+#   Writes compiled python code with proto messages and grpc services to
+#   $TEST_DRIVER_FULL_DIR/src/proto
+#######################################
+test_driver_compile_protos() {
+  declare -a protos
+  protos=(
+    "${TEST_DRIVER_PROTOS_PATH}/test.proto"
+    "${TEST_DRIVER_PROTOS_PATH}/messages.proto"
+    "${TEST_DRIVER_PROTOS_PATH}/empty.proto"
+  )
+  echo "Generate python code from grpc.testing protos: ${protos[*]}"
+  cd "${TEST_DRIVER_REPO_DIR}"
+  python -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out="${TEST_DRIVER_FULL_DIR}" \
+    --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \
+    "${protos[@]}"
+  local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}"
+  echo "Generated files ${protos_out_dir}:"
+  ls -Fl "${protos_out_dir}"
+}
+
+#######################################
+# Installs the test driver and it's requirements.
+# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation
+# Globals:
+#   TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing
+#                         the test driver
+#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+# Arguments:
+#   The directory for test driver's source code
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+test_driver_install() {
+  readonly TEST_DRIVER_REPO_DIR="${1:?Usage test_driver_install TEST_DRIVER_REPO_DIR}"
+  readonly TEST_DRIVER_FULL_DIR="${TEST_DRIVER_REPO_DIR}/${TEST_DRIVER_PATH}"
+  test_driver_get_source
+  test_driver_pip_install
+  test_driver_compile_protos
+}
+
+#######################################
+# Outputs Kokoro image version and Ubuntu's lsb_release
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output to stdout
+#######################################
+kokoro_print_version() {
+  echo "Kokoro VM version:"
+  if [[ -f /VERSION ]]; then
+    cat /VERSION
+  fi
+  run_ignore_exit_code lsb_release -a
+}
+
+#######################################
+# Report extra information about the job via sponge properties.
+# Globals:
+#   KOKORO_ARTIFACTS_DIR
+#   GIT_ORIGIN_URL
+#   GIT_COMMIT_SHORT
+#   TESTGRID_EXCLUDE
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output to stdout
+#   Writes job properties to $KOKORO_ARTIFACTS_DIR/custom_sponge_config.csv
+#######################################
+kokoro_write_sponge_properties() {
+  # CSV format: "property_name","property_value"
+  # Bump TESTS_FORMAT_VERSION when reported test name changed enough to when it
+  # makes more sense to discard previous test results from a testgrid board.
+  # Use GIT_ORIGIN_URL to exclude test runs executed against repo forks from
+  # testgrid reports.
+  cat >"${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" <<EOF
+TESTS_FORMAT_VERSION,2
+TESTGRID_EXCLUDE,${TESTGRID_EXCLUDE:-0}
+GIT_ORIGIN_URL,${GIT_ORIGIN_URL:?GIT_ORIGIN_URL must be set}
+GIT_COMMIT_SHORT,${GIT_COMMIT_SHORT:?GIT_COMMIT_SHORT must be set}
+EOF
+  echo "Sponge properties:"
+  cat "${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv"
+}
+
+#######################################
+# Configure Python virtual environment on Kokoro VM.
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `pyenv` commands to stdout
+#######################################
+kokoro_setup_python_virtual_environment() {
+  # Kokoro provides pyenv, so use it instead of `python -m venv`
+  echo "Setup pyenv environment"
+  eval "$(pyenv init -)"
+  eval "$(pyenv virtualenv-init -)"
+  py_latest_patch="$(pyenv versions --bare --skip-aliases | grep -E "^${PYTHON_VERSION}\.[0-9]{1,2}$" | sort --version-sort | tail -n 1)"
+  echo "Activating python ${py_latest_patch} virtual environment"
+  pyenv virtualenv "${py_latest_patch}" k8s_xds_test_runner
+  pyenv local k8s_xds_test_runner
+  pyenv activate k8s_xds_test_runner
+}
+
+#######################################
+# Installs and configures the test driver on Kokoro VM.
+# Globals:
+#   KOKORO_ARTIFACTS_DIR
+#   TEST_DRIVER_REPO_NAME
+#   SRC_DIR: Populated with absolute path to the source repo on Kokoro VM
+#   TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing
+#                         the test driver
+#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
+#   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+#   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
+#   GIT_COMMIT: Populated with the SHA-1 of git commit being built
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
+# Arguments:
+#   The name of github repository being built
+# Outputs:
+#   Writes the output to stdout, stderr, files
+#######################################
+kokoro_setup_test_driver() {
+  local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}"
+  # Capture Kokoro VM version info in the log.
+  kokoro_print_version
+
+  # Kokoro clones repo to ${KOKORO_ARTIFACTS_DIR}/github/${GITHUB_REPOSITORY}
+  local github_root="${KOKORO_ARTIFACTS_DIR}/github"
+  readonly SRC_DIR="${github_root}/${src_repository_name}"
+  local test_driver_repo_dir="${github_root}/${TEST_DRIVER_REPO_NAME}"
+  parse_src_repo_git_info SRC_DIR
+  kokoro_write_sponge_properties
+  kokoro_setup_python_virtual_environment
+
+  # gcloud requires python, so this should be executed after pyenv setup
+  gcloud_update
+  gcloud_get_cluster_credentials
+  test_driver_install "${test_driver_repo_dir}"
+  # shellcheck disable=SC2034  # Used in the main script
+  readonly TEST_DRIVER_FLAGFILE="config/grpc-testing.cfg"
+  # Test artifacts dir: xml reports, logs, etc.
+  local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts"
+  # Folders after $artifacts_dir reported as target name
+  readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}"
+  mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}"
+}
+
+#######################################
+# Installs and configures the test driver for testing build script locally.
+# Globals:
+#   TEST_DRIVER_REPO_NAME
+#   TEST_DRIVER_REPO_DIR: Unless provided, populated with a temporary dir with
+#                         the path to the test driver repo
+#   SRC_DIR: Populated with absolute path to the source repo
+#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
+#   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
+#   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
+#   GIT_COMMIT: Populated with the SHA-1 of git commit being built
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+# Arguments:
+#   The path to the folder containing the build script
+# Outputs:
+#   Writes the output to stdout, stderr, files
+#######################################
+local_setup_test_driver() {
+  local script_dir="${1:?Usage: local_setup_test_driver SCRIPT_DIR}"
+  readonly SRC_DIR="$(git -C "${script_dir}" rev-parse --show-toplevel)"
+  parse_src_repo_git_info SRC_DIR
+  readonly KUBE_CONTEXT="${KUBE_CONTEXT:-$(kubectl config current-context)}"
+  local test_driver_repo_dir
+  test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}"
+  test_driver_install "${test_driver_repo_dir}"
+  # shellcheck disable=SC2034  # Used in the main script
+  readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg"
+  # Test out
+  readonly TEST_XML_OUTPUT_DIR="${TEST_DRIVER_FULL_DIR}/out"
+  mkdir -p "${TEST_XML_OUTPUT_DIR}"
+}
+