From 48ce79f7e5db829da87d7cee2c9b0ddeb05d2af1 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 23 Jul 2021 15:26:13 -0700 Subject: [PATCH] Allow url-map tests to run concurrently (#26764) * Allow url-map tests to run concurrently * Clean-up client namespace if everything goes smoothly * Make isort happy * Update tools/internal_ci/linux/grpc_xds_url_map_python.sh Co-authored-by: Sergii Tkachenko * Add logging && change suffix generation * Move the suffix generation to helpers * Remove unused import Co-authored-by: Sergii Tkachenko --- tools/internal_ci/linux/grpc_xds_url_map.cfg | 2 +- .../linux/grpc_xds_url_map_python.cfg | 26 ++++ .../linux/grpc_xds_url_map_python.sh | 146 ++++++++++++++++++ .../xds_k8s_test_driver/config/url-map.cfg | 1 - .../framework/helpers/rand.py | 24 ++- .../framework/xds_k8s_testcase.py | 17 +- .../framework/xds_url_map_test_resources.py | 30 +++- 7 files changed, 222 insertions(+), 24 deletions(-) create mode 100644 tools/internal_ci/linux/grpc_xds_url_map_python.cfg create mode 100755 tools/internal_ci/linux/grpc_xds_url_map_python.sh diff --git a/tools/internal_ci/linux/grpc_xds_url_map.cfg b/tools/internal_ci/linux/grpc_xds_url_map.cfg index 42bfb1e861e..738269d2bc8 100644 --- a/tools/internal_ci/linux/grpc_xds_url_map.cfg +++ b/tools/internal_ci/linux/grpc_xds_url_map.cfg @@ -16,7 +16,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_xds_url_map.sh" -timeout_mins: 120 +timeout_mins: 60 action { define_artifacts { regex: "artifacts/**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_xds_url_map_python.cfg b/tools/internal_ci/linux/grpc_xds_url_map_python.cfg new file mode 100644 index 00000000000..685278441cc --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds_url_map_python.cfg @@ -0,0 +1,26 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_xds_url_map_python.sh" +timeout_mins: 60 +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*sponge_log.log" + strip_prefix: "artifacts" + } +} diff --git a/tools/internal_ci/linux/grpc_xds_url_map_python.sh b/tools/internal_ci/linux/grpc_xds_url_map_python.sh new file mode 100755 index 00000000000..59152f80985 --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds_url_map_python.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Constants +readonly GITHUB_REPOSITORY_NAME="grpc" +# GKE Cluster +readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a" +readonly GKE_CLUSTER_ZONE="us-central1-a" +## xDS test client Docker images +readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client" +readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" +readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" +readonly LANGUAGE_NAME="Python" + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# BUILD_APP_PATH +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# Arguments: +# None +# Outputs: +# Writes the output of `gcloud builds submit` to stdout, stderr +####################################### +build_test_app_docker_images() { + echo "Building ${LANGUAGE_NAME} xDS interop test app Docker images" + + pushd "${SRC_DIR}" + docker build \ + -f src/python/grpcio_tests/tests_py3_only/interop/Dockerfile.client \ + -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + . + + popd + + gcloud -q auth configure-docker + + docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" +} + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +build_docker_images_if_needed() { + # Check if images already exist + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 + if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${client_tags}" ]]; then + build_test_app_docker_images + else + echo "Skipping ${LANGUAGE_NAME} test app build" + fi +} + +####################################### +# Executes the test case +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +run_test() { + # Test driver usage: + # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + local test_name="${1:?Usage: run_test test_name}" + set -x + python -m "tests.${test_name}" \ + --flagfile="${TEST_DRIVER_FLAGFILE}" \ + --kube_context="${KUBE_CONTEXT}" \ + --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \ + --flagfile="config/url-map.cfg" + set +x +} + +####################################### +# Main function: provision software necessary to execute tests, and run them +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# None +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +main() { + local script_dir + script_dir="$(dirname "$0")" + # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh + source "${script_dir}/grpc_xds_k8s_install_test_driver.sh" + set -x + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "${script_dir}" + fi + build_docker_images_if_needed + # Run tests + cd "${TEST_DRIVER_FULL_DIR}" + run_test url_map +} + +main "$@" diff --git a/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg b/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg index 388197ea160..0f812567736 100644 --- a/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg +++ b/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg @@ -1,6 +1,5 @@ --resource_prefix=interop-psm-url-map --strategy=reuse -# TODO(lidiz): Remove the next line when xds port randomization supported. --server_xds_port=8848 # NOTE(lidiz) we pin the server image to java-server because: # 1. Only Java server understands the rpc-behavior metadata. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py index 3a593e9489f..dedf2143603 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py @@ -15,6 +15,8 @@ import random import string +import framework.helpers.datetime + # Alphanumeric characters, similar to regex [:alnum:] class, [a-zA-Z0-9] ALPHANUM = string.ascii_letters + string.digits # Lowercase alphanumeric characters: [a-z0-9] @@ -25,9 +27,23 @@ ALPHANUM_LOWERCASE = string.ascii_lowercase + string.digits def rand_string(length: int = 8, *, lowercase: bool = False) -> str: """Return random alphanumeric string of given length. - Space for default arguments: alphabet^length - lowercase and uppercase = (26*2 + 10)^8 = 2.18e14 = 218 trillion. - lowercase only = (26 + 10)^8 = 2.8e12 = 2.8 trillion. - """ + Space for default arguments: alphabet^length + lowercase and uppercase = (26*2 + 10)^8 = 2.18e14 = 218 trillion. + lowercase only = (26 + 10)^8 = 2.8e12 = 2.8 trillion. + """ alphabet = ALPHANUM_LOWERCASE if lowercase else ALPHANUM return ''.join(random.choices(population=alphabet, k=length)) + + +def random_resource_suffix() -> str: + """Return a ready-to-use resource suffix with datetime and nonce.""" + # Date and time suffix for debugging. Seconds skipped, not as relevant + # Format example: 20210626-1859 + datetime_suffix: str = framework.helpers.datetime.datetime_suffix() + # Use lowercase chars because some resource names won't allow uppercase. + # For len 5, total (26 + 10)^5 = 60,466,176 combinations. + # Approx. number of test runs needed to start at the same minute to + # produce a collision: math.sqrt(math.pi/2 * (26+10)**5) ≈ 9745. + # https://en.wikipedia.org/wiki/Birthday_attack#Mathematics + unique_hash: str = rand_string(5, lowercase=True) + return f'{datetime_suffix}-{unique_hash}' diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py index 337d8edf6a1..52e7d3a9b9b 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py @@ -26,7 +26,6 @@ from google.protobuf import json_format from framework import xds_flags from framework import xds_k8s_flags from framework.helpers import retryers -import framework.helpers.datetime import framework.helpers.rand from framework.infrastructure import gcp from framework.infrastructure import k8s @@ -130,7 +129,8 @@ class XdsKubernetesTestCase(absltest.TestCase, metaclass=abc.ABCMeta): super().setUp() if self._resource_suffix_randomize: - self.resource_suffix = self._random_resource_suffix() + self.resource_suffix = framework.helpers.rand.random_resource_suffix( + ) logger.info('Test run resource prefix: %s, suffix: %s', self.resource_prefix, self.resource_suffix) @@ -195,19 +195,6 @@ class XdsKubernetesTestCase(absltest.TestCase, metaclass=abc.ABCMeta): self.server_runner.cleanup(force=self.force_cleanup, force_namespace=self.force_cleanup) - @staticmethod - def _random_resource_suffix() -> str: - # Date and time suffix for debugging. Seconds skipped, not as relevant - # Format example: 20210626-1859 - datetime_suffix: str = framework.helpers.datetime.datetime_suffix() - # Use lowercase chars because some resource names won't allow uppercase. - # For len 5, total (26 + 10)^5 = 60,466,176 combinations. - # Approx. number of test runs needed to start at the same minute to - # produce a collision: math.sqrt(math.pi/2 * (26+10)**5) ≈ 9745. - # https://en.wikipedia.org/wiki/Birthday_attack#Mathematics - unique_hash: str = framework.helpers.rand.rand_string(5, lowercase=True) - return f'{datetime_suffix}-{unique_hash}' - def setupTrafficDirectorGrpc(self): self.td.setup_for_grpc(self.server_xds_host, self.server_xds_port, diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py index 88bb0bbd7ea..7f2fbdc5fdf 100644 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py +++ b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py @@ -15,6 +15,7 @@ import functools import inspect +import time from typing import Any, Iterable, List, Mapping, Tuple from absl import flags @@ -22,6 +23,7 @@ from absl import logging from framework import xds_flags from framework import xds_k8s_flags +import framework.helpers.rand from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director @@ -140,6 +142,17 @@ class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): if absl_flags is not None: for key in absl_flags: setattr(self, key, absl_flags[key]) + # Pick a client_namespace_suffix if not set + if self.resource_suffix is None: + self.resource_suffix = "" + self.client_namespace_suffix = framework.helpers.rand.random_resource_suffix( + ) + else: + self.client_namespace_suffix = self.resource_suffix + logging.info( + 'GcpResourceManager: resource prefix=%s, suffix=%s, client_namespace_suffix=%s', + self.resource_prefix, self.resource_suffix, + self.client_namespace_suffix) # API managers self.k8s_api_manager = k8s.KubernetesApiManager(self.kube_context) self.gcp_api_manager = gcp.api.GcpApiManager() @@ -153,9 +166,16 @@ class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): # Kubernetes namespace self.k8s_namespace = k8s.KubernetesNamespace(self.k8s_api_manager, self.resource_prefix) + if self.client_namespace_suffix != self.resource_suffix: + self.k8s_client_namespace = k8s.KubernetesNamespace( + self.k8s_api_manager, + client_app.KubernetesClientRunner.make_namespace_name( + self.resource_prefix, self.client_namespace_suffix)) + else: + self.k8s_client_namespace = self.k8s_namespace # Kubernetes Test Client self.test_client_runner = client_app.KubernetesClientRunner( - self.k8s_namespace, + self.k8s_client_namespace, deployment_name=self.client_name, image_name=self.client_image, gcp_project=self.project, @@ -203,6 +223,7 @@ class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): logging.info('GcpResourceManager: pre clean-up') self.td.cleanup(force=True) self.test_client_runner.delete_namespace() + self.test_server_runner.delete_namespace() def setup(self, test_case_classes: 'Iterable[XdsUrlMapTestCase]') -> None: if self.strategy not in ['create', 'keep']: @@ -276,6 +297,11 @@ class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): self.td.wait_for_affinity_backends_healthy_status() def cleanup(self) -> None: + if hasattr(self, 'test_client_runner'): + self.test_client_runner.cleanup( + force=True, + # Clean-up ephemeral client namespace + force_namespace=self.k8s_client_namespace != self.k8s_namespace) if self.strategy not in ['create']: logging.info( 'GcpResourceManager: skipping tear down for strategy [%s]', @@ -284,8 +310,6 @@ class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): logging.info('GcpResourceManager: start tear down') if hasattr(self, 'td'): self.td.cleanup(force=True) - if hasattr(self, 'test_client_runner'): - self.test_client_runner.cleanup(force=True) if hasattr(self, 'test_server_runner'): self.test_server_runner.cleanup(force=True) if hasattr(self, 'test_server_alternative_runner'):