diff --git a/black.toml b/black.toml index 55087061282..1469b60d05b 100644 --- a/black.toml +++ b/black.toml @@ -30,7 +30,6 @@ line_length = 80 src_paths = [ "examples/python/data_transmission", "examples/python/async_streaming", - "tools/run_tests/xds_k8s_test_driver", "src/python/grpcio_tests", "tools/run_tests", ] diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index 76a25c7dd85..41a1364e0d3 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -29,14 +29,11 @@ DIRS=( 'src/python/grpcio_testing/grpc_testing' 'src/python/grpcio_status/grpc_status' 'src/python/grpcio_observability/grpc_observability' - 'tools/run_tests/xds_k8s_test_driver/bin' - 'tools/run_tests/xds_k8s_test_driver/framework' ) TEST_DIRS=( 'src/python/grpcio_tests/tests' 'src/python/grpcio_tests/tests_gevent' - 'tools/run_tests/xds_k8s_test_driver/tests' ) VIRTUALENV=python_pylint_venv diff --git a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh index 76a5eaa5e44..4e226fae5e0 100644 --- a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh +++ b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh @@ -284,7 +284,7 @@ test_driver_compile_protos() { ####################################### # Installs the test driver and it's requirements. -# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation +# https://github.com/grpc/psm-interop#installation # Globals: # TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing # the test driver diff --git a/tools/internal_ci/linux/grpc_xds_k8s_lb.sh b/tools/internal_ci/linux/grpc_xds_k8s_lb.sh index e2c8daa82d8..b0fe742e6f8 100755 --- a/tools/internal_ci/linux/grpc_xds_k8s_lb.sh +++ b/tools/internal_ci/linux/grpc_xds_k8s_lb.sh @@ -99,7 +99,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh b/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh index 10c1525fcf2..23047a17209 100755 --- a/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh +++ b/tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh @@ -112,7 +112,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/grpc_xds_k8s_run_xtest.sh b/tools/internal_ci/linux/grpc_xds_k8s_run_xtest.sh index 6f2a1c92614..2fa8cfdc8d1 100755 --- a/tools/internal_ci/linux/grpc_xds_k8s_run_xtest.sh +++ b/tools/internal_ci/linux/grpc_xds_k8s_run_xtest.sh @@ -69,7 +69,7 @@ run_test() { exit 1 fi # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local client_lang="$1" local client_branch="$2" local server_lang="$3" diff --git a/tools/internal_ci/linux/grpc_xds_url_map.sh b/tools/internal_ci/linux/grpc_xds_url_map.sh index 324494d7579..17f66a326e1 100755 --- a/tools/internal_ci/linux/grpc_xds_url_map.sh +++ b/tools/internal_ci/linux/grpc_xds_url_map.sh @@ -87,7 +87,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/grpc_xds_url_map_python.sh b/tools/internal_ci/linux/grpc_xds_url_map_python.sh index 541d5875dfa..0bf795f4bac 100755 --- a/tools/internal_ci/linux/grpc_xds_url_map_python.sh +++ b/tools/internal_ci/linux/grpc_xds_url_map_python.sh @@ -97,7 +97,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/psm-csm.sh b/tools/internal_ci/linux/psm-csm.sh index ecdf113006b..68079bb1928 100755 --- a/tools/internal_ci/linux/psm-csm.sh +++ b/tools/internal_ci/linux/psm-csm.sh @@ -99,7 +99,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/psm-security-python.sh b/tools/internal_ci/linux/psm-security-python.sh index d746b79c8f5..cddfa085589 100755 --- a/tools/internal_ci/linux/psm-security-python.sh +++ b/tools/internal_ci/linux/psm-security-python.sh @@ -114,7 +114,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/internal_ci/linux/psm-security.sh b/tools/internal_ci/linux/psm-security.sh index 48c006ffce0..2cebf9da0b4 100755 --- a/tools/internal_ci/linux/psm-security.sh +++ b/tools/internal_ci/linux/psm-security.sh @@ -99,7 +99,7 @@ build_docker_images_if_needed() { ####################################### run_test() { # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + # https://github.com/grpc/psm-interop#basic-usage local test_name="${1:?Usage: run_test test_name}" local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" mkdir -pv "${out_dir}" diff --git a/tools/run_tests/xds_k8s_test_driver/.gitignore b/tools/run_tests/xds_k8s_test_driver/.gitignore deleted file mode 100644 index d16ec129981..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -config/local-*.cfg -src/proto -venv/ -venv-*/ -out/ diff --git a/tools/run_tests/xds_k8s_test_driver/README.md b/tools/run_tests/xds_k8s_test_driver/README.md index 51a01670d95..a9fb5944e2a 100644 --- a/tools/run_tests/xds_k8s_test_driver/README.md +++ b/tools/run_tests/xds_k8s_test_driver/README.md @@ -1,458 +1,3 @@ # xDS Kubernetes Interop Tests -Proxyless Security Mesh Interop Tests executed on Kubernetes. - -### Experimental -Work in progress. Internal APIs may and will change. Please refrain from making -changes to this codebase at the moment. - -### Stabilization roadmap -- [x] Replace retrying with tenacity -- [x] Generate namespace for each test to prevent resource name conflicts and - allow running tests in parallel -- [x] Security: run server and client in separate namespaces -- [ ] Make framework.infrastructure.gcp resources [first-class - citizen](https://en.wikipedia.org/wiki/First-class_citizen), support - simpler CRUD -- [x] Security: manage `roles/iam.workloadIdentityUser` role grant lifecycle for - dynamically-named namespaces -- [x] Restructure `framework.test_app` and `framework.xds_k8s*` into a module - containing xDS-interop-specific logic -- [ ] Address inline TODOs in code -- [x] Improve README.md documentation, explain helpers in bin/ folder - -## Installation - -#### Requirements -1. Python v3.9+ -2. [Google Cloud SDK](https://cloud.google.com/sdk/docs/install) -3. `kubectl` - -`kubectl` can be installed via `gcloud components install kubectl`, or system package manager: https://kubernetes.io/docs/tasks/tools/#kubectl - -Python3 venv tool may need to be installed from APT on some Ubuntu systems: -```shell -sudo apt-get install python3-venv -``` - -##### Getting Started - -1. If you haven't, [initialize](https://cloud.google.com/sdk/docs/install-sdk) gcloud SDK -2. Activate gcloud [configuration](https://cloud.google.com/sdk/docs/configurations) with your project -3. Enable gcloud services: - ```shell - gcloud services enable \ - compute.googleapis.com \ - container.googleapis.com \ - logging.googleapis.com \ - monitoring.googleapis.com \ - networksecurity.googleapis.com \ - networkservices.googleapis.com \ - secretmanager.googleapis.com \ - trafficdirector.googleapis.com - ``` - -#### Configure GKE cluster -This is an example outlining minimal requirements to run the [baseline tests](#xds-baseline-tests). -Update gloud sdk: -```shell -gcloud -q components update -``` - -Pre-populate environment variables for convenience. To find project id, refer to -[Identifying projects](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). -```shell -export PROJECT_ID="your-project-id" -export PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" --format="value(projectNumber)") -# Compute Engine default service account -export GCE_SA="${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" -# The prefix to name GCP resources used by the framework -export RESOURCE_PREFIX="xds-k8s-interop-tests" - -# The zone name your cluster, f.e. xds-k8s-test-cluster -export CLUSTER_NAME="${RESOURCE_PREFIX}-cluster" -# The zone of your cluster, f.e. us-central1-a -export ZONE="us-central1-a" -# Dedicated GCP Service Account to use with workload identity. -export WORKLOAD_SA_NAME="${RESOURCE_PREFIX}" -export WORKLOAD_SA_EMAIL="${WORKLOAD_SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" -``` - -##### Create the cluster -Minimal requirements: [VPC-native](https://cloud.google.com/traffic-director/docs/security-proxyless-setup) -cluster with [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) enabled. -```shell -gcloud container clusters create "${CLUSTER_NAME}" \ - --scopes=cloud-platform \ - --zone="${ZONE}" \ - --enable-ip-alias \ - --workload-pool="${PROJECT_ID}.svc.id.goog" \ - --workload-metadata=GKE_METADATA \ - --tags=allow-health-checks -``` -For security tests you also need to create CAs and configure the cluster to use those CAs -as described -[here](https://cloud.google.com/traffic-director/docs/security-proxyless-setup#configure-cas). - -##### Create the firewall rule -Allow [health checking mechanisms](https://cloud.google.com/traffic-director/docs/set-up-proxyless-gke#creating_the_health_check_firewall_rule_and_backend_service) -to query the workloads health. -This step can be skipped, if the driver is executed with `--ensure_firewall`. -```shell -gcloud compute firewall-rules create "${RESOURCE_PREFIX}-allow-health-checks" \ - --network=default --action=allow --direction=INGRESS \ - --source-ranges="35.191.0.0/16,130.211.0.0/22" \ - --target-tags=allow-health-checks \ - --rules=tcp:8080-8100 -``` - -##### Setup GCP Service Account - -Create dedicated GCP Service Account to use -with [workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity). - -```shell -gcloud iam service-accounts create "${WORKLOAD_SA_NAME}" \ - --display-name="xDS K8S Interop Tests Workload Identity Service Account" -``` - -Enable the service account to [access the Traffic Director API](https://cloud.google.com/traffic-director/docs/prepare-for-envoy-setup#enable-service-account). -```shell -gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ - --member="serviceAccount:${WORKLOAD_SA_EMAIL}" \ - --role="roles/trafficdirector.client" -``` - -##### Allow access to images -The test framework needs read access to the client and server images and the bootstrap -generator image. You may have these images in your project but if you want to use these -from the grpc-testing project you will have to grant the necessary access to these images -using https://cloud.google.com/container-registry/docs/access-control#grant or a -gsutil command. For example, to grant access to images stored in `grpc-testing` project GCR, run: - -```sh -gsutil iam ch "serviceAccount:${GCE_SA}:objectViewer" gs://artifacts.grpc-testing.appspot.com/ -``` - -##### Allow test driver to configure workload identity automatically -Test driver will automatically grant `roles/iam.workloadIdentityUser` to -allow the Kubernetes service account to impersonate the dedicated GCP workload -service account (corresponds to the step 5 -of [Authenticating to Google Cloud](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#authenticating_to)). -This action requires the test framework to have `iam.serviceAccounts.create` -permission on the project. - -If you're running test framework locally, and you have `roles/owner` to your -project, **you can skip this step**. -If you're configuring the test framework to run on a CI: use `roles/owner` -account once to allow test framework to grant `roles/iam.workloadIdentityUser`. - -```shell -# Assuming CI is using Compute Engine default service account. -gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ - --member="serviceAccount:${GCE_SA}" \ - --role="roles/iam.serviceAccountAdmin" \ - --condition-from-file=<(cat <<-END ---- -title: allow_workload_identity_only -description: Restrict serviceAccountAdmin to granting role iam.workloadIdentityUser -expression: |- - api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []) - .hasOnly(['roles/iam.workloadIdentityUser']) -END -) -``` - -##### Configure GKE cluster access -```shell -# Unless you're using GCP VM with preconfigured Application Default Credentials, acquire them for your user -gcloud auth application-default login - -# Install authentication plugin for kubectl. -# Details: https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke -gcloud components install gke-gcloud-auth-plugin - -# Configuring GKE cluster access for kubectl -gcloud container clusters get-credentials "${CLUSTER_NAME}" --zone "${ZONE}" - -# Save generated kube context name -export KUBE_CONTEXT="$(kubectl config current-context)" -``` - -#### Install python dependencies - -```shell -# Create python virtual environment -python3 -m venv venv - -# Activate virtual environment -. ./venv/bin/activate - -# Install requirements -pip install -r requirements.lock - -# Generate protos -python -m grpc_tools.protoc --proto_path=../../../ \ - --python_out=. --grpc_python_out=. \ - src/proto/grpc/testing/empty.proto \ - src/proto/grpc/testing/messages.proto \ - src/proto/grpc/testing/test.proto -``` - -# Basic usage - -## Local development -This test driver allows running tests locally against remote GKE clusters, right -from your dev environment. You need: - -1. Follow [installation](#installation) instructions -2. Authenticated `gcloud` -3. `kubectl` context (see [Configure GKE cluster access](#configure-gke-cluster-access)) -4. Run tests with `--debug_use_port_forwarding` argument. The test driver - will automatically start and stop port forwarding using - `kubectl` subprocesses. (experimental) - -### Making changes to the driver -1. Install additional dev packages: `pip install -r requirements-dev.txt` -2. Use `./bin/black.sh` and `./bin/isort.sh` helpers to auto-format code. - -### Updating Python Dependencies - -We track our Python-level dependencies using three different files: - -- `requirements.txt` -- `dev-requirements.txt` -- `requirements.lock` - -`requirements.txt` lists modules without specific versions supplied, though -versions ranges may be specified. `requirements.lock` is generated from -`requirements.txt` and _does_ specify versions for every dependency in the -transitive dependency tree. - -When updating `requirements.txt`, you must also update `requirements.lock`. To -do this, navigate to this directory and run `./bin/freeze.sh`. - -### Setup test configuration - -There are many arguments to be passed into the test run. You can save the -arguments to a config file ("flagfile") for your development environment. -Use [`config/local-dev.cfg.example`](https://github.com/grpc/grpc/blob/master/tools/run_tests/xds_k8s_test_driver/config/local-dev.cfg.example) -as a starting point: - -```shell -cp config/local-dev.cfg.example config/local-dev.cfg -``` - -If you exported environment variables in the above sections, you can -template them into the local config (note this recreates the config): - -```shell -envsubst < config/local-dev.cfg.example > config/local-dev.cfg -``` - -Learn more about flagfiles in [abseil documentation](https://abseil.io/docs/python/guides/flags#a-note-about---flagfile). - -## Test suites - -See the full list of available test suites in the [`tests/`](https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver/tests) folder. - -### xDS Baseline Tests - -Test suite meant to confirm that basic xDS features work as expected. Executing -it before other test suites will help to identify whether test failure related -to specific features under test, or caused by unrelated infrastructure -disturbances. - -```shell -# Help -python -m tests.baseline_test --help -python -m tests.baseline_test --helpfull - -# Run the baseline test with local-dev.cfg settings -python -m tests.baseline_test --flagfile="config/local-dev.cfg" - -# Same as above, but using the helper script -./run.sh tests/baseline_test.py -``` - -### xDS Security Tests -Test suite meant to verify mTLS/TLS features. Note that this requires -additional environment configuration. For more details, and for the -setup for the security tests, see -["Setting up Traffic Director service security with proxyless gRPC"](https://cloud.google.com/traffic-director/docs/security-proxyless-setup) user guide. - -```shell -# Run the security test with local-dev.cfg settings -python -m tests.security_test --flagfile="config/local-dev.cfg" - -# Same as above, but using the helper script -./run.sh tests/security_test.py -``` - -## Helper scripts -You can use interop xds-k8s [`bin/`](https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver/bin) -scripts to configure TD, start k8s instances step-by-step, and keep them alive -for as long as you need. - -* To run helper scripts using local config: - * `python -m bin.script_name --flagfile=config/local-dev.cfg` - * `./run.sh bin/script_name.py` automatically appends the flagfile -* Use `--help` to see script-specific argument -* Use `--helpfull` to see all available argument - -#### Overview -```shell -# Helper tool to configure Traffic Director with different security options -python -m bin.run_td_setup --help - -# Helper tools to run the test server, client (with or without security) -python -m bin.run_test_server --help -python -m bin.run_test_client --help - -# Helper tool to verify different security configurations via channelz -python -m bin.run_channelz --help -``` - -#### `./run.sh` helper -Use `./run.sh` to execute helper scripts and tests with `config/local-dev.cfg`. - -```sh -USAGE: ./run.sh script_path [arguments] - script_path: path to python script to execute, relative to driver root folder - arguments ...: arguments passed to program in sys.argv - -ENVIRONMENT: - XDS_K8S_CONFIG: file path to the config flagfile, relative to - driver root folder. Default: config/local-dev.cfg - Will be appended as --flagfile="config_absolute_path" argument - XDS_K8S_DRIVER_VENV_DIR: the path to python virtual environment directory - Default: $XDS_K8S_DRIVER_DIR/venv -DESCRIPTION: -This tool performs the following: -1) Ensures python virtual env installed and activated -2) Exports test driver root in PYTHONPATH -3) Automatically appends --flagfile="\$XDS_K8S_CONFIG" argument - -EXAMPLES: -./run.sh bin/run_td_setup.py --help -./run.sh bin/run_td_setup.py --helpfull -XDS_K8S_CONFIG=./path-to-flagfile.cfg ./run.sh bin/run_td_setup.py --resource_suffix=override-suffix -./run.sh tests/baseline_test.py -./run.sh tests/security_test.py --verbosity=1 --logger_levels=__main__:DEBUG,framework:DEBUG -./run.sh tests/security_test.py SecurityTest.test_mtls --nocheck_local_certs -``` - -## Partial setups -### Regular workflow -```shell -# Setup Traffic Director -./run.sh bin/run_td_setup.py - -# Start test server -./run.sh bin/run_test_server.py - -# Add test server to the backend service -./run.sh bin/run_td_setup.py --cmd=backends-add - -# Start test client -./run.sh bin/run_test_client.py -``` - -### Secure workflow -```shell -# Setup Traffic Director in mtls. See --help for all options -./run.sh bin/run_td_setup.py --security=mtls - -# Start test server in a secure mode -./run.sh bin/run_test_server.py --mode=secure - -# Add test server to the backend service -./run.sh bin/run_td_setup.py --cmd=backends-add - -# Start test client in a secure more --mode=secure -./run.sh bin/run_test_client.py --mode=secure -``` - -### Sending RPCs -#### Start port forwarding -```shell -# Client: all services always on port 8079 -kubectl port-forward deployment.apps/psm-grpc-client 8079 - -# Server regular mode: all grpc services on port 8080 -kubectl port-forward deployment.apps/psm-grpc-server 8080 -# OR -# Server secure mode: TestServiceImpl is on 8080, -kubectl port-forward deployment.apps/psm-grpc-server 8080 -# everything else (channelz, healthcheck, CSDS) on 8081 -kubectl port-forward deployment.apps/psm-grpc-server 8081 -``` - -#### Send RPCs with grpccurl -```shell -# 8081 if security enabled -export SERVER_ADMIN_PORT=8080 - -# List server services using reflection -grpcurl --plaintext 127.0.0.1:$SERVER_ADMIN_PORT list -# List client services using reflection -grpcurl --plaintext 127.0.0.1:8079 list - -# List channels via channelz -grpcurl --plaintext 127.0.0.1:$SERVER_ADMIN_PORT grpc.channelz.v1.Channelz.GetTopChannels -grpcurl --plaintext 127.0.0.1:8079 grpc.channelz.v1.Channelz.GetTopChannels - -# Send GetClientStats to the client -grpcurl --plaintext -d '{"num_rpcs": 10, "timeout_sec": 30}' 127.0.0.1:8079 \ - grpc.testing.LoadBalancerStatsService.GetClientStats -``` - -### Cleanup -* First, make sure to stop port forwarding, if any -* Run `./bin/cleanup.sh` - -##### Partial cleanup -You can run commands below to stop/start, create/delete resources however you want. -Generally, it's better to remove resources in the opposite order of their creation. - -Cleanup regular resources: -```shell -# Cleanup TD resources -./run.sh bin/run_td_setup.py --cmd=cleanup -# Stop test client -./run.sh bin/run_test_client.py --cmd=cleanup -# Stop test server, and remove the namespace -./run.sh bin/run_test_server.py --cmd=cleanup --cleanup_namespace -``` - -Cleanup regular and security-specific resources: -```shell -# Cleanup TD resources, with security -./run.sh bin/run_td_setup.py --cmd=cleanup --security=mtls -# Stop test client (secure) -./run.sh bin/run_test_client.py --cmd=cleanup --mode=secure -# Stop test server (secure), and remove the namespace -./run.sh bin/run_test_server.py --cmd=cleanup --cleanup_namespace --mode=secure -``` - -In addition, here's some other helpful partial cleanup commands: -```shell -# Remove all backends from the backend services -./run.sh bin/run_td_setup.py --cmd=backends-cleanup - -# Stop the server, but keep the namespace -./run.sh bin/run_test_server.py --cmd=cleanup --nocleanup_namespace -``` - -### Known errors -#### Error forwarding port -If you stopped a test with `ctrl+c`, while using `--debug_use_port_forwarding`, -you might see an error like this: - -> `framework.infrastructure.k8s.PortForwardingError: Error forwarding port, unexpected output Unable to listen on port 8081: Listeners failed to create with the following errors: [unable to create listener: Error listen tcp4 127.0.0.1:8081: bind: address already in use]` - -Unless you're running `kubectl port-forward` manually, it's likely that `ctrl+c` -interrupted python before it could clean up subprocesses. - -You can do `ps aux | grep port-forward` and then kill the processes by id, -or with `killall kubectl` +The source is migrated to https://github.com/grpc/psm-interop. diff --git a/tools/run_tests/xds_k8s_test_driver/bin/__init__.py b/tools/run_tests/xds_k8s_test_driver/bin/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/bin/black.sh b/tools/run_tests/xds_k8s_test_driver/bin/black.sh deleted file mode 100755 index 60c077d375a..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/black.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -display_usage() { - cat </dev/stderr -A helper to run black formatter. - -USAGE: $0 [--diff] - --diff: Do not apply changes, only show the diff - --check: Do not apply changes, only print what files will be changed - -ENVIRONMENT: - XDS_K8S_DRIVER_VENV_DIR: the path to python virtual environment directory - Default: $XDS_K8S_DRIVER_DIR/venv -EXAMPLES: -$0 -$0 --diff -$0 --check -EOF - exit 1 -} - -if [[ "$1" == "-h" || "$1" == "--help" ]]; then - display_usage -fi - -SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -readonly SCRIPT_DIR -readonly XDS_K8S_DRIVER_DIR="${SCRIPT_DIR}/.." - -cd "${XDS_K8S_DRIVER_DIR}" - -# Relative paths not yet supported by shellcheck. -# shellcheck source=/dev/null -source "${XDS_K8S_DRIVER_DIR}/bin/ensure_venv.sh" - -if [[ "$1" == "--diff" ]]; then - readonly MODE="--diff" -elif [[ "$1" == "--check" ]]; then - readonly MODE="--check" -else - readonly MODE="" -fi - -# shellcheck disable=SC2086 -exec python -m black --config=../../../black.toml ${MODE} . diff --git a/tools/run_tests/xds_k8s_test_driver/bin/cleanup.sh b/tools/run_tests/xds_k8s_test_driver/bin/cleanup.sh deleted file mode 100755 index 98b7f837be0..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/cleanup.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -display_usage() { - cat </dev/stderr -Performs full TD and K8S resource cleanup - -USAGE: $0 [--nosecure] [arguments] - --nosecure: Skip cleanup for the resources specific for PSM Security - arguments ...: additional arguments passed to ./run.sh - -ENVIRONMENT: - XDS_K8S_CONFIG: file path to the config flagfile, relative to - driver root folder. Default: config/local-dev.cfg - Will be appended as --flagfile="config_absolute_path" argument - XDS_K8S_DRIVER_VENV_DIR: the path to python virtual environment directory - Default: $XDS_K8S_DRIVER_DIR/venv -EXAMPLES: -$0 -$0 --nosecure -XDS_K8S_CONFIG=./path-to-flagfile.cfg $0 --resource_suffix=override-suffix -EOF - exit 1 -} - -if [[ "$1" == "-h" || "$1" == "--help" ]]; then - display_usage -fi - -SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -readonly SCRIPT_DIR -readonly XDS_K8S_DRIVER_DIR="${SCRIPT_DIR}/.." - -cd "${XDS_K8S_DRIVER_DIR}" - -if [[ "$1" == "--nosecure" ]]; then - shift - ./run.sh bin/run_td_setup.py --cmd=cleanup "$@" && \ - ./run.sh bin/run_test_client.py --cmd=cleanup --cleanup_namespace "$@" && \ - ./run.sh bin/run_test_server.py --cmd=cleanup --cleanup_namespace "$@" -else - ./run.sh bin/run_td_setup.py --cmd=cleanup --security=mtls "$@" && \ - ./run.sh bin/run_test_client.py --cmd=cleanup --cleanup_namespace --mode=secure "$@" && \ - ./run.sh bin/run_test_server.py --cmd=cleanup --cleanup_namespace --mode=secure "$@" -fi diff --git a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/README.md b/tools/run_tests/xds_k8s_test_driver/bin/cleanup/README.md deleted file mode 100644 index d6f636f35cd..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# This folder contains scripts to delete leaked resources from test runs - diff --git a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/cleanup.py b/tools/run_tests/xds_k8s_test_driver/bin/cleanup/cleanup.py deleted file mode 100755 index a778610eee5..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/cleanup.py +++ /dev/null @@ -1,714 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Clean up resources created by the tests. - -This is intended as a tool to delete leaked resources from old tests. - -Typical usage examples: - -python3 -m bin.cleanup.cleanup \ - --project=grpc-testing \ - --network=default-vpc \ - --kube_context=gke_grpc-testing_us-central1-a_psm-interop-security -""" -import dataclasses -import datetime -import functools -import json -import logging -import os -import re -import subprocess -import sys -from typing import Any, Callable, List, Optional - -from absl import app -from absl import flags -import dateutil - -from framework import xds_flags -from framework import xds_k8s_flags -from framework.helpers import retryers -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.infrastructure import traffic_director -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -Json = Any -_KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - -GCLOUD = os.environ.get("GCLOUD", "gcloud") -GCLOUD_CMD_TIMEOUT_S = datetime.timedelta(seconds=5).total_seconds() - -# Skip known k8s system namespaces. -K8S_PROTECTED_NAMESPACES = { - "default", - "gke-managed-system", - "kube-node-lease", - "kube-public", - "kube-system", -} - -# TODO(sergiitk): these should be flags. -LEGACY_DRIVER_ZONE = "us-central1-a" -LEGACY_DRIVER_SECONDARY_ZONE = "us-west1-b" - -PSM_INTEROP_PREFIX = "psm-interop" # Prefix for gke resources to delete. -URL_MAP_TEST_PREFIX = ( - "interop-psm-url-map" # Prefix for url-map test resources to delete. -) - -KEEP_PERIOD_HOURS = flags.DEFINE_integer( - "keep_hours", - default=48, - help=( - "number of hours for a resource to keep. Resources older than this will" - " be deleted. Default is 48 hours (2 days)" - ), -) -DRY_RUN = flags.DEFINE_bool( - "dry_run", - default=False, - help="dry run, print resources but do not perform deletion", -) -TD_RESOURCE_PREFIXES = flags.DEFINE_list( - "td_resource_prefixes", - default=[PSM_INTEROP_PREFIX], - help=( - "a comma-separated list of prefixes for which the leaked TD resources" - " will be deleted" - ), -) -SERVER_PREFIXES = flags.DEFINE_list( - "server_prefixes", - default=[PSM_INTEROP_PREFIX], - help=( - "a comma-separated list of prefixes for which the leaked servers will" - " be deleted" - ), -) -CLIENT_PREFIXES = flags.DEFINE_list( - "client_prefixes", - default=[PSM_INTEROP_PREFIX, URL_MAP_TEST_PREFIX], - help=( - "a comma-separated list of prefixes for which the leaked clients will" - " be deleted" - ), -) -MODE = flags.DEFINE_enum( - "mode", - default="td", - enum_values=["k8s", "td", "td_no_legacy"], - help="Mode: Kubernetes or Traffic Director", -) -SECONDARY = flags.DEFINE_bool( - "secondary", - default=False, - help="Cleanup secondary (alternative) resources", -) - -# The cleanup script performs some API calls directly, so some flags normally -# required to configure framework properly, are not needed here. -flags.FLAGS.set_default("resource_prefix", "ignored-by-cleanup") -flags.FLAGS.set_default("td_bootstrap_image", "ignored-by-cleanup") -flags.FLAGS.set_default("server_image", "ignored-by-cleanup") -flags.FLAGS.set_default("client_image", "ignored-by-cleanup") - - -@dataclasses.dataclass(eq=False) -class CleanupResult: - error_count: int = 0 - error_messages: List[str] = dataclasses.field(default_factory=list) - - def add_error(self, msg: str): - self.error_count += 1 - self.error_messages.append(f" {self.error_count}. {msg}") - - def format_messages(self): - return "\n".join(self.error_messages) - - -@dataclasses.dataclass(frozen=True) -class K8sResourceRule: - # regex to match - expression: str - # function to delete the resource - cleanup_ns_fn: Callable - - -# Global state, holding the result of the whole operation. -_CLEANUP_RESULT = CleanupResult() - - -def load_keep_config() -> None: - global KEEP_CONFIG - json_path = os.path.realpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "keep_xds_interop_resources.json", - ) - ) - with open(json_path, "r") as f: - KEEP_CONFIG = json.load(f) - logging.debug( - "Resource keep config loaded: %s", json.dumps(KEEP_CONFIG, indent=2) - ) - - -def is_marked_as_keep_gce(suffix: str) -> bool: - return suffix in KEEP_CONFIG["gce_framework"]["suffix"] - - -def is_marked_as_keep_gke(suffix: str) -> bool: - return suffix in KEEP_CONFIG["gke_framework"]["suffix"] - - -@functools.lru_cache() -def get_expire_timestamp() -> datetime.datetime: - return datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( - hours=KEEP_PERIOD_HOURS.value - ) - - -def exec_gcloud(project: str, *cmds: str) -> Json: - cmds = [GCLOUD, "--project", project, "--quiet"] + list(cmds) - if "list" in cmds: - # Add arguments to shape the list output - cmds.extend( - [ - "--format", - "json", - "--filter", - f"creationTimestamp <= {get_expire_timestamp().isoformat()}", - ] - ) - # Executing the gcloud command - logging.debug("Executing: %s", " ".join(cmds)) - proc = subprocess.Popen( - cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - # NOTE(lidiz) the gcloud subprocess won't return unless its output is read - stdout = proc.stdout.read() - stderr = proc.stderr.read() - try: - returncode = proc.wait(timeout=GCLOUD_CMD_TIMEOUT_S) - except subprocess.TimeoutExpired: - logging.error("> Timeout executing cmd [%s]", " ".join(cmds)) - return None - if returncode: - logging.error( - "> Failed to execute cmd [%s], returned %d, stderr: %s", - " ".join(cmds), - returncode, - stderr, - ) - return None - if stdout: - return json.loads(stdout) - return None - - -def cleanup_legacy_driver_resources(*, project: str, suffix: str, **kwargs): - """Removing GCP resources created by run_xds_tests.py.""" - # Unused, but kept for compatibility with cleanup_td_for_gke. - del kwargs - logging.info( - "----- Removing run_xds_tests.py resources with suffix [%s]", suffix - ) - exec_gcloud( - project, - "compute", - "forwarding-rules", - "delete", - f"test-forwarding-rule{suffix}", - "--global", - ) - exec_gcloud( - project, - "compute", - "target-http-proxies", - "delete", - f"test-target-proxy{suffix}", - ) - exec_gcloud( - project, - "alpha", - "compute", - "target-grpc-proxies", - "delete", - f"test-target-proxy{suffix}", - ) - exec_gcloud(project, "compute", "url-maps", "delete", f"test-map{suffix}") - exec_gcloud( - project, - "compute", - "backend-services", - "delete", - f"test-backend-service{suffix}", - "--global", - ) - exec_gcloud( - project, - "compute", - "backend-services", - "delete", - f"test-backend-service-alternate{suffix}", - "--global", - ) - exec_gcloud( - project, - "compute", - "backend-services", - "delete", - f"test-backend-service-extra{suffix}", - "--global", - ) - exec_gcloud( - project, - "compute", - "backend-services", - "delete", - f"test-backend-service-more-extra{suffix}", - "--global", - ) - exec_gcloud( - project, "compute", "firewall-rules", "delete", f"test-fw-rule{suffix}" - ) - exec_gcloud( - project, "compute", "health-checks", "delete", f"test-hc{suffix}" - ) - exec_gcloud( - project, - "compute", - "instance-groups", - "managed", - "delete", - f"test-ig{suffix}", - "--zone", - LEGACY_DRIVER_ZONE, - ) - exec_gcloud( - project, - "compute", - "instance-groups", - "managed", - "delete", - f"test-ig-same-zone{suffix}", - "--zone", - LEGACY_DRIVER_ZONE, - ) - exec_gcloud( - project, - "compute", - "instance-groups", - "managed", - "delete", - f"test-ig-secondary-zone{suffix}", - "--zone", - LEGACY_DRIVER_SECONDARY_ZONE, - ) - exec_gcloud( - project, - "compute", - "instance-templates", - "delete", - f"test-template{suffix}", - ) - - -# cleanup_td creates TrafficDirectorManager (and its varients for security and -# AppNet), and then calls the cleanup() methods. -# -# Note that the varients are all based on the basic TrafficDirectorManager, so -# their `cleanup()` might do duplicate work. But deleting an non-exist resource -# returns 404, and is OK. -def cleanup_td_for_gke(*, project, prefix, suffix, network): - gcp_api_manager = gcp.api.GcpApiManager() - plain_td = traffic_director.TrafficDirectorManager( - gcp_api_manager, - project=project, - network=network, - resource_prefix=prefix, - resource_suffix=suffix, - ) - security_td = traffic_director.TrafficDirectorSecureManager( - gcp_api_manager, - project=project, - network=network, - resource_prefix=prefix, - resource_suffix=suffix, - ) - # TODO: cleanup appnet resources. - # appnet_td = traffic_director.TrafficDirectorAppNetManager( - # gcp_api_manager, - # project=project, - # network=network, - # resource_prefix=resource_prefix, - # resource_suffix=resource_suffix) - - logger.info( - "----- Removing traffic director for gke, prefix %s, suffix %s", - prefix, - suffix, - ) - security_td.cleanup(force=True) - # appnet_td.cleanup(force=True) - plain_td.cleanup(force=True) - - -# cleanup_client creates a client runner, and calls its cleanup() method. -def cleanup_client( - project, - network, - k8s_api_manager, - client_namespace, - gcp_api_manager, - gcp_service_account, - *, - suffix: Optional[str] = "", -): - deployment_name = xds_flags.CLIENT_NAME.value - if suffix: - deployment_name = f"{deployment_name}-{suffix}" - - ns = k8s.KubernetesNamespace(k8s_api_manager, client_namespace) - # Shorten the timeout to avoid waiting for the stuck namespaces. - # Normal ns deletion during the cleanup takes less two minutes. - ns.wait_for_namespace_deleted_timeout_sec = 5 * 60 - client_runner = _KubernetesClientRunner( - k8s_namespace=ns, - deployment_name=deployment_name, - gcp_project=project, - network=network, - gcp_service_account=gcp_service_account, - gcp_api_manager=gcp_api_manager, - image_name="", - td_bootstrap_image="", - ) - - logger.info("Cleanup client") - try: - client_runner.cleanup(force=True, force_namespace=True) - except retryers.RetryError as err: - logger.error( - "Timeout waiting for namespace %s deletion. " - "Failed resource status:\n\n%s", - ns.name, - ns.pretty_format_status(err.result()), - ) - raise - - -# cleanup_server creates a server runner, and calls its cleanup() method. -def cleanup_server( - project, - network, - k8s_api_manager, - server_namespace, - gcp_api_manager, - gcp_service_account, - *, - suffix: Optional[str] = "", -): - deployment_name = xds_flags.SERVER_NAME.value - if suffix: - deployment_name = f"{deployment_name}-{suffix}" - - ns = k8s.KubernetesNamespace(k8s_api_manager, server_namespace) - # Shorten the timeout to avoid waiting for the stuck namespaces. - # Normal ns deletion during the cleanup takes less two minutes. - ns.wait_for_namespace_deleted_timeout_sec = 5 * 60 - server_runner = _KubernetesServerRunner( - k8s_namespace=ns, - deployment_name=deployment_name, - gcp_project=project, - network=network, - gcp_service_account=gcp_service_account, - gcp_api_manager=gcp_api_manager, - image_name="", - td_bootstrap_image="", - ) - - logger.info("Cleanup server") - try: - server_runner.cleanup(force=True, force_namespace=True) - except retryers.RetryError as err: - logger.error( - "Timeout waiting for namespace %s deletion. " - "Failed resource status:\n\n%s", - ns.name, - ns.pretty_format_status(err.result()), - ) - raise - - -def delete_leaked_td_resources( - dry_run, td_resource_rules, project, network, resources -): - for resource in resources: - logger.info("-----") - logger.info("----- Cleaning up resource %s", resource["name"]) - if dry_run: - # Skip deletion for dry-runs - logging.info("----- Skipped [Dry Run]: %s", resource["name"]) - continue - matched = False - for regex, resource_prefix, keep, remove_fn in td_resource_rules: - result = re.search(regex, resource["name"]) - if result is not None: - matched = True - if keep(result.group(1)): - logging.info("Skipped [keep]:") - break # break inner loop, continue outer loop - remove_fn( - project=project, - prefix=resource_prefix, - suffix=result.group(1), - network=network, - ) - break - if not matched: - logging.info( - "----- Skipped [does not matching resource name templates]" - ) - - -def delete_k8s_resources( - dry_run, - k8s_resource_rules, - project, - network, - k8s_api_manager, - gcp_service_account, - namespaces, -): - gcp_api_manager = gcp.api.GcpApiManager() - for ns in namespaces: - namespace_name: str = ns.metadata.name - if namespace_name in K8S_PROTECTED_NAMESPACES: - continue - - logger.info("-----") - logger.info("----- Cleaning up k8s namespaces %s", namespace_name) - - if ns.metadata.creation_timestamp > get_expire_timestamp(): - logging.info( - "----- Skipped [resource is within expiry date]: %s", - namespace_name, - ) - continue - - if dry_run: - # Skip deletion for dry-runs - logging.info("----- Skipped [Dry Run]: %s", ns.metadata.name) - continue - - rule: K8sResourceRule = _rule_match_k8s_namespace( - namespace_name, k8s_resource_rules - ) - if not rule: - logging.info( - "----- Skipped [does not matching resource name templates]: %s", - namespace_name, - ) - continue - - # Cleaning up. - try: - rule.cleanup_ns_fn( - project, - network, - k8s_api_manager, - namespace_name, - gcp_api_manager, - gcp_service_account, - suffix=("alt" if SECONDARY.value else None), - ) - except k8s.NotFound: - logging.warning("----- Skipped [not found]: %s", namespace_name) - except retryers.RetryError as err: - _CLEANUP_RESULT.add_error( - "Retries exhausted while waiting for the " - f"deletion of namespace {namespace_name}: " - f"{err}" - ) - logging.exception( - "----- Skipped [cleanup timed out]: %s", namespace_name - ) - except Exception as err: # noqa pylint: disable=broad-except - _CLEANUP_RESULT.add_error( - "Unexpected error while deleting " - f"namespace {namespace_name}: {err}" - ) - logging.exception( - "----- Skipped [cleanup unexpected error]: %s", namespace_name - ) - - logger.info("-----") - - -def _rule_match_k8s_namespace( - namespace_name: str, k8s_resource_rules: List[K8sResourceRule] -) -> Optional[K8sResourceRule]: - for rule in k8s_resource_rules: - result = re.search(rule.expression, namespace_name) - if result is not None: - return rule - return None - - -def find_and_remove_leaked_k8s_resources( - dry_run, project, network, gcp_service_account, k8s_context -): - k8s_resource_rules: List[K8sResourceRule] = [] - for prefix in CLIENT_PREFIXES.value: - k8s_resource_rules.append( - K8sResourceRule(f"{prefix}-client-(.*)", cleanup_client) - ) - for prefix in SERVER_PREFIXES.value: - k8s_resource_rules.append( - K8sResourceRule(f"{prefix}-server-(.*)", cleanup_server) - ) - - # Delete leaked k8s namespaces, those usually mean there are leaked testing - # client/servers from the gke framework. - k8s_api_manager = k8s.KubernetesApiManager(k8s_context) - nss = k8s_api_manager.core.list_namespace() - delete_k8s_resources( - dry_run, - k8s_resource_rules, - project, - network, - k8s_api_manager, - gcp_service_account, - nss.items, - ) - - -def find_and_remove_leaked_td_resources(dry_run, project, network): - cleanup_legacy: bool = MODE.value != "td_no_legacy" - td_resource_rules = [ - # itmes in each tuple, in order - # - regex to match - # - prefix of the resource (only used by gke resources) - # - function to check of the resource should be kept - # - function to delete the resource - ] - - if cleanup_legacy: - td_resource_rules += [ - ( - r"test-hc(.*)", - "", - is_marked_as_keep_gce, - cleanup_legacy_driver_resources, - ), - ( - r"test-template(.*)", - "", - is_marked_as_keep_gce, - cleanup_legacy_driver_resources, - ), - ] - - for prefix in TD_RESOURCE_PREFIXES.value: - td_resource_rules.append( - ( - f"{prefix}-health-check-(.*)", - prefix, - is_marked_as_keep_gke, - cleanup_td_for_gke, - ), - ) - - # List resources older than KEEP_PERIOD. We only list health-checks and - # instance templates because these are leaves in the resource dependency - # tree. - # - # E.g. forwarding-rule depends on the target-proxy. So leaked - # forwarding-rule indicates there's a leaked target-proxy (because this - # target proxy cannot deleted unless the forwarding rule is deleted). The - # leaked target-proxy is guaranteed to be a super set of leaked - # forwarding-rule. - compute = gcp.compute.ComputeV1(gcp.api.GcpApiManager(), project) - leaked_health_checks = [] - for item in compute.list_health_check()["items"]: - if ( - dateutil.parser.isoparse(item["creationTimestamp"]) - <= get_expire_timestamp() - ): - leaked_health_checks.append(item) - - delete_leaked_td_resources( - dry_run, td_resource_rules, project, network, leaked_health_checks - ) - - # Delete leaked instance templates, those usually mean there are leaked VMs - # from the gce framework. Also note that this is only needed for the gce - # resources. - if cleanup_legacy: - leaked_instance_templates = exec_gcloud( - project, "compute", "instance-templates", "list" - ) - delete_leaked_td_resources( - dry_run, - td_resource_rules, - project, - network, - leaked_instance_templates, - ) - - -def main(argv): - # TODO(sergiitk): instead, base on absltest so that result.xml is available. - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - load_keep_config() - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - project: str = xds_flags.PROJECT.value - network: str = xds_flags.NETWORK.value - gcp_service_account: str = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value - dry_run: bool = DRY_RUN.value - k8s_context: str = xds_k8s_flags.KUBE_CONTEXT.value - - if MODE.value == "td" or MODE.value == "td_no_legacy": - find_and_remove_leaked_td_resources(dry_run, project, network) - elif MODE.value == "k8s": - # 'unset' value is used in td-only mode to bypass the validation - # for the required flag. - assert k8s_context != "unset" - find_and_remove_leaked_k8s_resources( - dry_run, project, network, gcp_service_account, k8s_context - ) - - logger.info("##################### Done cleaning up #####################") - if _CLEANUP_RESULT.error_count > 0: - logger.error( - "Cleanup failed for %i resource(s). Errors: [\n%s\n].\n" - "Please inspect the log files for stack traces corresponding " - "to these errors.", - _CLEANUP_RESULT.error_count, - _CLEANUP_RESULT.format_messages(), - ) - sys.exit(1) - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/keep_xds_interop_resources.json b/tools/run_tests/xds_k8s_test_driver/bin/cleanup/keep_xds_interop_resources.json deleted file mode 100644 index b55a7699242..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/cleanup/keep_xds_interop_resources.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "gce_framework": { - "suffix": [] - }, - "gke_framework": { - "suffix": [] - } -} \ No newline at end of file diff --git a/tools/run_tests/xds_k8s_test_driver/bin/cleanup_cluster.sh b/tools/run_tests/xds_k8s_test_driver/bin/cleanup_cluster.sh deleted file mode 100755 index 5755a46721c..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/cleanup_cluster.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -readonly SCRIPT_DIR -readonly XDS_K8S_DRIVER_DIR="${SCRIPT_DIR}/.." - -cd "${XDS_K8S_DRIVER_DIR}" - -NO_SECURE="yes" -DATE_TO=$(date -Iseconds) - -while [[ $# -gt 0 ]]; do - case $1 in - --secure) NO_SECURE=""; shift ;; - --date_to=*) DATE_TO="${1#*=}T00:00:00Z"; shift ;; - *) echo "Unknown argument $1"; exit 1 ;; - esac -done - -jq_selector=$(cat <<- 'EOM' - .items[].metadata | - select( - (.name | test("-(client|server)-")) and - (.creationTimestamp < $date_to) - ) | .name -EOM -) - -mapfile -t namespaces < <(\ - kubectl get namespaces --sort-by='{.metadata.creationTimestamp}'\ - --selector='owner=xds-k8s-interop-test'\ - -o json\ - | jq --arg date_to "${DATE_TO}" -r "${jq_selector}" -) - -if [[ -z "${namespaces[*]}" ]]; then - echo "All clean." - exit 0 -fi - -echo "Found namespaces:" -namespaces_joined=$(IFS=,; printf '%s' "${namespaces[*]}") -kubectl get namespaces --sort-by='{.metadata.creationTimestamp}' \ - --selector="name in (${namespaces_joined})" - -# Suffixes -mapfile -t suffixes < <(\ - printf '%s\n' "${namespaces[@]}" | sed -E 's/^.+-(server|client)-//' -) -echo -echo "Found suffixes: ${suffixes[*]}" -echo "Count: ${#namespaces[@]}" - -echo "Run plan:" -for suffix in "${suffixes[@]}"; do - echo ./bin/cleanup.sh ${NO_SECURE:+"--nosecure"} "--resource_suffix=${suffix}" -done - -read -r -n 1 -p "Continue? (y/N) " answer -if [[ "$answer" != "${answer#[Yy]}" ]] ;then - echo - echo "Starting the cleanup." -else - echo - echo "Exit" - exit 0 -fi - -failed=0 -for suffix in "${suffixes[@]}"; do - echo "-------------------- Cleaning suffix ${suffix} --------------------" - set -x - ./bin/cleanup.sh ${NO_SECURE:+"--nosecure"} "--resource_suffix=${suffix}" || (( ++failed )) - set +x - echo "-------------------- Finished cleaning ${suffix} --------------------" -done -echo "Failed runs: ${failed}" -if (( failed > 0 )); then - exit 1 -fi diff --git a/tools/run_tests/xds_k8s_test_driver/bin/ensure_venv.sh b/tools/run_tests/xds_k8s_test_driver/bin/ensure_venv.sh deleted file mode 100755 index 288c3fac90d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/ensure_venv.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Expected $XDS_K8S_DRIVER_DIR to be set by the file sourcing this. -readonly XDS_K8S_DRIVER_VENV_DIR="${XDS_K8S_DRIVER_VENV_DIR:-$XDS_K8S_DRIVER_DIR/venv}" - -if [[ -z "${VIRTUAL_ENV}" ]]; then - if [[ -d "${XDS_K8S_DRIVER_VENV_DIR}" ]]; then - # Intentional: No need to check python venv activate script. - # shellcheck source=/dev/null - source "${XDS_K8S_DRIVER_VENV_DIR}/bin/activate" - else - echo "Missing python virtual environment directory: ${XDS_K8S_DRIVER_VENV_DIR}" >&2 - echo "Follow README.md installation steps first." >&2 - exit 1 - fi -fi diff --git a/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh b/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh deleted file mode 100755 index 5cbd6f016ea..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -exo pipefail - -VENV_NAME="venv-$(mktemp -d)" -readonly VENV_NAME - -python3 -m virtualenv "${VENV_NAME}" - -"${VENV_NAME}"/bin/pip install -r requirements.txt -"${VENV_NAME}"/bin/pip freeze --require-virtualenv --local -r requirements.txt \ - > requirements.lock - -rm -rf "${VENV_NAME}" diff --git a/tools/run_tests/xds_k8s_test_driver/bin/isort.sh b/tools/run_tests/xds_k8s_test_driver/bin/isort.sh deleted file mode 100755 index 3acd70092f1..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/isort.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -display_usage() { - cat </dev/stderr -A helper to run isort import sorter. - -USAGE: $0 [--diff] - --diff: Do not apply changes, only show the diff - -ENVIRONMENT: - XDS_K8S_DRIVER_VENV_DIR: the path to python virtual environment directory - Default: $XDS_K8S_DRIVER_DIR/venv -EXAMPLES: -$0 -$0 --diff -EOF - exit 1 -} - -if [[ "$1" == "-h" || "$1" == "--help" ]]; then - display_usage -fi - -SCRIPT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -readonly SCRIPT_DIR -readonly XDS_K8S_DRIVER_DIR="${SCRIPT_DIR}/.." - -cd "${XDS_K8S_DRIVER_DIR}" - -# Relative paths not yet supported by shellcheck. -# shellcheck source=/dev/null -source "${XDS_K8S_DRIVER_DIR}/bin/ensure_venv.sh" - -if [[ "$1" == "--diff" ]]; then - readonly MODE="--diff" -else - readonly MODE="--overwrite-in-place" -fi - -# typing is the only module allowed to put imports on the same line: -# https://google.github.io/styleguide/pyguide.html#313-imports-formatting -exec python -m isort "${MODE}" \ - --settings-path=../../../black.toml \ - framework bin tests - diff --git a/tools/run_tests/xds_k8s_test_driver/bin/lib/__init__.py b/tools/run_tests/xds_k8s_test_driver/bin/lib/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/lib/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/bin/lib/common.py b/tools/run_tests/xds_k8s_test_driver/bin/lib/common.py deleted file mode 100755 index 3c93c2fbede..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/lib/common.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Common functionality for bin/ python helpers.""" -import atexit -import signal -import sys - -from absl import logging - -from framework import xds_flags -from framework import xds_k8s_flags -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.test_app import client_app -from framework.test_app import server_app -from framework.test_app.runners.k8s import gamma_server_runner -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.get_absl_logger() - -# Type aliases -KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner -GammaServerRunner = gamma_server_runner.GammaServerRunner -_XdsTestServer = server_app.XdsTestServer -_XdsTestClient = client_app.XdsTestClient - - -def make_client_namespace( - k8s_api_manager: k8s.KubernetesApiManager, - namespace_name: str = None, -) -> k8s.KubernetesNamespace: - if not namespace_name: - namespace_name: str = KubernetesClientRunner.make_namespace_name( - xds_flags.RESOURCE_PREFIX.value, xds_flags.RESOURCE_SUFFIX.value - ) - return k8s.KubernetesNamespace(k8s_api_manager, namespace_name) - - -def make_client_runner( - namespace: k8s.KubernetesNamespace, - gcp_api_manager: gcp.api.GcpApiManager, - *, - port_forwarding: bool = False, - reuse_namespace: bool = True, - enable_workload_identity: bool = True, - mode: str = "default", -) -> KubernetesClientRunner: - # KubernetesClientRunner arguments. - runner_kwargs = dict( - deployment_name=xds_flags.CLIENT_NAME.value, - image_name=xds_k8s_flags.CLIENT_IMAGE.value, - td_bootstrap_image=xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value, - gcp_project=xds_flags.PROJECT.value, - gcp_api_manager=gcp_api_manager, - gcp_service_account=xds_k8s_flags.GCP_SERVICE_ACCOUNT.value, - xds_server_uri=xds_flags.XDS_SERVER_URI.value, - network=xds_flags.NETWORK.value, - stats_port=xds_flags.CLIENT_PORT.value, - reuse_namespace=reuse_namespace, - debug_use_port_forwarding=port_forwarding, - enable_workload_identity=enable_workload_identity, - ) - - if mode == "secure": - runner_kwargs.update( - deployment_template="client-secure.deployment.yaml" - ) - return KubernetesClientRunner(namespace, **runner_kwargs) - - -def make_server_namespace( - k8s_api_manager: k8s.KubernetesApiManager, - server_runner: KubernetesServerRunner = KubernetesServerRunner, -) -> k8s.KubernetesNamespace: - namespace_name: str = server_runner.make_namespace_name( - xds_flags.RESOURCE_PREFIX.value, xds_flags.RESOURCE_SUFFIX.value - ) - return k8s.KubernetesNamespace(k8s_api_manager, namespace_name) - - -def make_server_runner( - namespace: k8s.KubernetesNamespace, - gcp_api_manager: gcp.api.GcpApiManager, - *, - port_forwarding: bool = False, - reuse_namespace: bool = True, - reuse_service: bool = False, - enable_workload_identity: bool = True, - mode: str = "default", -) -> KubernetesServerRunner: - # KubernetesServerRunner arguments. - runner_kwargs = dict( - deployment_name=xds_flags.SERVER_NAME.value, - image_name=xds_k8s_flags.SERVER_IMAGE.value, - td_bootstrap_image=xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value, - xds_server_uri=xds_flags.XDS_SERVER_URI.value, - gcp_project=xds_flags.PROJECT.value, - gcp_api_manager=gcp_api_manager, - gcp_service_account=xds_k8s_flags.GCP_SERVICE_ACCOUNT.value, - network=xds_flags.NETWORK.value, - reuse_namespace=reuse_namespace, - reuse_service=reuse_service, - debug_use_port_forwarding=port_forwarding, - enable_workload_identity=enable_workload_identity, - ) - - server_runner = KubernetesServerRunner - if mode == "secure": - runner_kwargs["deployment_template"] = "server-secure.deployment.yaml" - elif mode == "gamma": - server_runner = GammaServerRunner - - return server_runner(namespace, **runner_kwargs) - - -def _ensure_atexit(signum, frame): - """Needed to handle signals or atexit handler won't be called.""" - del frame - - # Pylint is wrong about "Module 'signal' has no 'Signals' member": - # https://docs.python.org/3/library/signal.html#signal.Signals - sig = signal.Signals(signum) # pylint: disable=no-member - logger.warning("Caught %r, initiating graceful shutdown...\n", sig) - sys.exit(1) - - -def _graceful_exit( - server_runner: KubernetesServerRunner, client_runner: KubernetesClientRunner -): - """Stop port forwarding processes.""" - client_runner.stop_pod_dependencies() - server_runner.stop_pod_dependencies() - - -def register_graceful_exit( - server_runner: KubernetesServerRunner, client_runner: KubernetesClientRunner -): - atexit.register(_graceful_exit, server_runner, client_runner) - for signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): - signal.signal(signum, _ensure_atexit) - - -def get_client_pod( - client_runner: KubernetesClientRunner, deployment_name: str -) -> k8s.V1Pod: - client_deployment: k8s.V1Deployment - client_deployment = client_runner.k8s_namespace.get_deployment( - deployment_name - ) - client_pod_name: str = client_runner._wait_deployment_pod_count( - client_deployment - )[0] - return client_runner._wait_pod_started(client_pod_name) - - -def get_server_pod( - server_runner: KubernetesServerRunner, deployment_name: str -) -> k8s.V1Pod: - server_deployment: k8s.V1Deployment - server_deployment = server_runner.k8s_namespace.get_deployment( - deployment_name - ) - server_pod_name: str = server_runner._wait_deployment_pod_count( - server_deployment - )[0] - return server_runner._wait_pod_started(server_pod_name) - - -def get_test_server_for_pod( - server_runner: KubernetesServerRunner, server_pod: k8s.V1Pod, **kwargs -) -> _XdsTestServer: - return server_runner._xds_test_server_for_pod(server_pod, **kwargs) - - -def get_test_client_for_pod( - client_runner: KubernetesClientRunner, client_pod: k8s.V1Pod, **kwargs -) -> _XdsTestClient: - return client_runner._xds_test_client_for_pod(client_pod, **kwargs) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_channelz.py b/tools/run_tests/xds_k8s_test_driver/bin/run_channelz.py deleted file mode 100755 index 62ab103a0d6..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_channelz.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Channelz debugging tool for xDS test client/server. - -This is intended as a debugging / local development helper and not executed -as a part of interop test suites. - -Typical usage examples: - - # Show channel and server socket pair - python -m bin.run_channelz --flagfile=config/local-dev.cfg - - # Evaluate setup for different security configurations - python -m bin.run_channelz --flagfile=config/local-dev.cfg --security=tls - python -m bin.run_channelz --flagfile=config/local-dev.cfg --security=mtls_error - - # More information and usage options - python -m bin.run_channelz --helpfull -""" -import hashlib - -from absl import app -from absl import flags -from absl import logging - -from bin.lib import common -from framework import xds_flags -from framework import xds_k8s_flags -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.rpc import grpc_channelz -from framework.test_app import client_app -from framework.test_app import server_app - -# Flags -_SECURITY = flags.DEFINE_enum( - "security", - default=None, - enum_values=[ - "mtls", - "tls", - "plaintext", - "mtls_error", - "server_authz_error", - ], - help="Show info for a security setup", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) -# Running outside of a test suite, so require explicit resource_suffix. -flags.mark_flag_as_required(xds_flags.RESOURCE_SUFFIX.name) -flags.register_validator( - xds_flags.SERVER_XDS_PORT.name, - lambda val: val > 0, - message=( - "Run outside of a test suite, must provide" - " the exact port value (must be greater than 0)." - ), -) - -logger = logging.get_absl_logger() - -# Type aliases -_Channel = grpc_channelz.Channel -_Socket = grpc_channelz.Socket -_ChannelState = grpc_channelz.ChannelState -_XdsTestServer = server_app.XdsTestServer -_XdsTestClient = client_app.XdsTestClient - - -def debug_cert(cert): - if not cert: - return "" - sha1 = hashlib.sha1(cert) - return f"sha1={sha1.hexdigest()}, len={len(cert)}" - - -def debug_sock_tls(tls): - return ( - f"local: {debug_cert(tls.local_certificate)}\n" - f"remote: {debug_cert(tls.remote_certificate)}" - ) - - -def get_deployment_pods(k8s_ns, deployment_name): - deployment = k8s_ns.get_deployment(deployment_name) - return k8s_ns.list_deployment_pods(deployment) - - -def debug_security_setup_negative(test_client): - """Debug negative cases: mTLS Error, Server AuthZ error - - 1) mTLS Error: Server expects client mTLS cert, - but client configured only for TLS. - 2) AuthZ error: Client does not authorize server because of mismatched - SAN name. - """ - # Client side. - client_correct_setup = True - channel: _Channel = test_client.wait_for_server_channel_state( - state=_ChannelState.TRANSIENT_FAILURE - ) - try: - subchannel, *subchannels = list( - test_client.channelz.list_channel_subchannels(channel) - ) - except ValueError: - print( - "Client setup fail: subchannel not found. " - "Common causes: test client didn't connect to TD; " - "test client exhausted retries, and closed all subchannels." - ) - return - - # Client must have exactly one subchannel. - logger.debug("Found subchannel, %s", subchannel) - if subchannels: - client_correct_setup = False - print(f"Unexpected subchannels {subchannels}") - subchannel_state: _ChannelState = subchannel.data.state.state - if subchannel_state is not _ChannelState.TRANSIENT_FAILURE: - client_correct_setup = False - print( - "Subchannel expected to be in " - "TRANSIENT_FAILURE, same as its channel" - ) - - # Client subchannel must have no sockets. - sockets = list(test_client.channelz.list_subchannels_sockets(subchannel)) - if sockets: - client_correct_setup = False - print(f"Unexpected subchannel sockets {sockets}") - - # Results. - if client_correct_setup: - print( - "Client setup pass: the channel " - "to the server has exactly one subchannel " - "in TRANSIENT_FAILURE, and no sockets" - ) - - -def debug_security_setup_positive(test_client, test_server): - """Debug positive cases: mTLS, TLS, Plaintext.""" - test_client.wait_for_server_channel_ready() - client_sock: _Socket = test_client.get_active_server_channel_socket() - server_sock: _Socket = test_server.get_server_socket_matching_client( - client_sock - ) - - server_tls = server_sock.security.tls - client_tls = client_sock.security.tls - - print(f"\nServer certs:\n{debug_sock_tls(server_tls)}") - print(f"\nClient certs:\n{debug_sock_tls(client_tls)}") - print() - - if server_tls.local_certificate: - eq = server_tls.local_certificate == client_tls.remote_certificate - print(f"(TLS) Server local matches client remote: {eq}") - else: - print("(TLS) Not detected") - - if server_tls.remote_certificate: - eq = server_tls.remote_certificate == client_tls.local_certificate - print(f"(mTLS) Server remote matches client local: {eq}") - else: - print("(mTLS) Not detected") - - -def debug_basic_setup(test_client, test_server): - """Show channel and server socket pair""" - test_client.wait_for_server_channel_ready() - client_sock: _Socket = test_client.get_active_server_channel_socket() - server_sock: _Socket = test_server.get_server_socket_matching_client( - client_sock - ) - - logger.debug("Client socket: %s\n", client_sock) - logger.debug("Matching server socket: %s\n", server_sock) - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - # Flags. - should_port_forward: bool = xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value - enable_workload_identity: bool = ( - xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value - ) - is_secure: bool = bool(_SECURITY.value) - - # Setup. - gcp_api_manager = gcp.api.GcpApiManager() - k8s_api_manager = k8s.KubernetesApiManager(xds_k8s_flags.KUBE_CONTEXT.value) - - # Server. - server_namespace = common.make_server_namespace(k8s_api_manager) - server_runner = common.make_server_runner( - server_namespace, - gcp_api_manager, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - mode="secure", - ) - # Find server pod. - server_pod: k8s.V1Pod = common.get_server_pod( - server_runner, xds_flags.SERVER_NAME.value - ) - - # Client - client_namespace = common.make_client_namespace(k8s_api_manager) - client_runner = common.make_client_runner( - client_namespace, - gcp_api_manager, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - mode="secure", - ) - # Find client pod. - client_pod: k8s.V1Pod = common.get_client_pod( - client_runner, xds_flags.CLIENT_NAME.value - ) - - # Ensure port forwarding stopped. - common.register_graceful_exit(server_runner, client_runner) - - # Create server app for the server pod. - test_server: _XdsTestServer = common.get_test_server_for_pod( - server_runner, - server_pod, - test_port=xds_flags.SERVER_PORT.value, - secure_mode=is_secure, - ) - test_server.set_xds_address( - xds_flags.SERVER_XDS_HOST.value, xds_flags.SERVER_XDS_PORT.value - ) - - # Create client app for the client pod. - test_client: _XdsTestClient = common.get_test_client_for_pod( - client_runner, client_pod, server_target=test_server.xds_uri - ) - - with test_client, test_server: - if _SECURITY.value in ("mtls", "tls", "plaintext"): - debug_security_setup_positive(test_client, test_server) - elif _SECURITY.value in ("mtls_error", "server_authz_error"): - debug_security_setup_negative(test_client) - else: - debug_basic_setup(test_client, test_server) - - logger.info("SUCCESS!") - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py b/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py deleted file mode 100755 index d811a16a378..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_ping_pong.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from absl import app -from absl import flags -from absl import logging - -from bin.lib import common -from framework import xds_flags -from framework import xds_k8s_flags -from framework.helpers import grpc as helpers_grpc -import framework.helpers.highlighter -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.rpc import grpc_channelz -from framework.rpc import grpc_testing -from framework.test_app import client_app -from framework.test_app import server_app - -# Flags -_MODE = flags.DEFINE_enum( - "mode", - default="default", - enum_values=["default", "secure", "gamma"], - help="Select a deployment of the client/server", -) -_NUM_RPCS = flags.DEFINE_integer( - "num_rpcs", - default=100, - lower_bound=1, - upper_bound=10_000, - help="The number of RPCs to check.", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) -# Running outside of a test suite, so require explicit resource_suffix. -flags.mark_flag_as_required(xds_flags.RESOURCE_SUFFIX.name) -flags.register_validator( - xds_flags.SERVER_XDS_PORT.name, - lambda val: val > 0, - message=( - "Run outside of a test suite, must provide" - " the exact port value (must be greater than 0)." - ), -) - -logger = logging.get_absl_logger() - -# Type aliases -_Channel = grpc_channelz.Channel -_Socket = grpc_channelz.Socket -_ChannelState = grpc_channelz.ChannelState -_XdsTestServer = server_app.XdsTestServer -_XdsTestClient = client_app.XdsTestClient -LoadBalancerStatsResponse = grpc_testing.LoadBalancerStatsResponse - - -def get_client_rpc_stats( - test_client: _XdsTestClient, num_rpcs: int -) -> LoadBalancerStatsResponse: - lb_stats = test_client.get_load_balancer_stats(num_rpcs=num_rpcs) - hl = framework.helpers.highlighter.HighlighterYaml() - logger.info( - "[%s] Received LoadBalancerStatsResponse:\n%s", - test_client.hostname, - hl.highlight(helpers_grpc.lb_stats_pretty(lb_stats)), - ) - return lb_stats - - -def run_ping_pong(test_client: _XdsTestClient, num_rpcs: int): - test_client.wait_for_active_xds_channel() - test_client.wait_for_server_channel_ready() - lb_stats = get_client_rpc_stats(test_client, num_rpcs) - for backend, rpcs_count in lb_stats.rpcs_by_peer.items(): - if int(rpcs_count) < 1: - raise AssertionError( - f"Backend {backend} did not receive a single RPC" - ) - - failed = int(lb_stats.num_failures) - if int(lb_stats.num_failures) > 0: - raise AssertionError( - f"Expected all RPCs to succeed: {failed} of {num_rpcs} failed" - ) - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - # Flags. - should_port_forward: bool = xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value - enable_workload_identity: bool = ( - xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value - ) - - # Setup. - gcp_api_manager = gcp.api.GcpApiManager() - k8s_api_manager = k8s.KubernetesApiManager(xds_k8s_flags.KUBE_CONTEXT.value) - - # Server. - server_namespace = common.make_server_namespace(k8s_api_manager) - server_runner = common.make_server_runner( - server_namespace, - gcp_api_manager, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - mode=_MODE.value, - ) - # Find server pod. - server_pod: k8s.V1Pod = common.get_server_pod( - server_runner, xds_flags.SERVER_NAME.value - ) - - # Client - client_namespace = common.make_client_namespace(k8s_api_manager) - client_runner = common.make_client_runner( - client_namespace, - gcp_api_manager, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - mode=_MODE.value, - ) - # Find client pod. - client_pod: k8s.V1Pod = common.get_client_pod( - client_runner, xds_flags.CLIENT_NAME.value - ) - - # Ensure port forwarding stopped. - common.register_graceful_exit(server_runner, client_runner) - - # Create server app for the server pod. - test_server: _XdsTestServer = common.get_test_server_for_pod( - server_runner, - server_pod, - test_port=xds_flags.SERVER_PORT.value, - secure_mode=_MODE.value == "secure", - ) - test_server.set_xds_address( - xds_flags.SERVER_XDS_HOST.value, xds_flags.SERVER_XDS_PORT.value - ) - - # Create client app for the client pod. - test_client: _XdsTestClient = common.get_test_client_for_pod( - client_runner, client_pod, server_target=test_server.xds_uri - ) - - with test_client, test_server: - run_ping_pong(test_client, _NUM_RPCS.value) - - logger.info("SUCCESS!") - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_td_setup.py b/tools/run_tests/xds_k8s_test_driver/bin/run_td_setup.py deleted file mode 100755 index 2c250b16197..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_td_setup.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Configure Traffic Director for different GRPC Proxyless. - -This is intended as a debugging / local development helper and not executed -as a part of interop test suites. - -Typical usage examples: - - # Regular proxyless setup - python -m bin.run_td_setup --flagfile=config/local-dev.cfg - - # Additional commands: cleanup, backend management, etc. - python -m bin.run_td_setup --flagfile=config/local-dev.cfg --cmd=cleanup - - # PSM security setup options: mtls, tls, etc. - python -m bin.run_td_setup --flagfile=config/local-dev.cfg --security=mtls - - # More information and usage options - python -m bin.run_td_setup --helpfull -""" -import logging - -from absl import app -from absl import flags - -from framework import xds_flags -from framework import xds_k8s_flags -from framework.helpers import rand -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.infrastructure import traffic_director -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -# Flags -_CMD = flags.DEFINE_enum( - "cmd", - default="create", - enum_values=[ - "cycle", - "create", - "cleanup", - "backends-add", - "backends-cleanup", - "unused-xds-port", - ], - help="Command", -) -_SECURITY = flags.DEFINE_enum( - "security", - default=None, - enum_values=[ - "mtls", - "tls", - "plaintext", - "mtls_error", - "server_authz_error", - ], - help="Configure TD with security", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) -# Running outside of a test suite, so require explicit resource_suffix. -flags.mark_flag_as_required(xds_flags.RESOURCE_SUFFIX.name) - - -@flags.multi_flags_validator( - (xds_flags.SERVER_XDS_PORT.name, _CMD.name), - message=( - "Run outside of a test suite, must provide" - " the exact port value (must be greater than 0)." - ), -) -def _check_server_xds_port_flag(flags_dict): - if flags_dict[_CMD.name] not in ("create", "cycle"): - return True - return flags_dict[xds_flags.SERVER_XDS_PORT.name] > 0 - - -# Type aliases -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - - -def main( - argv, -): # pylint: disable=too-many-locals,too-many-branches,too-many-statements - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - command = _CMD.value - security_mode = _SECURITY.value - - project: str = xds_flags.PROJECT.value - network: str = xds_flags.NETWORK.value - - # Resource names. - resource_prefix: str = xds_flags.RESOURCE_PREFIX.value - resource_suffix: str = xds_flags.RESOURCE_SUFFIX.value - - # Test server - server_name = xds_flags.SERVER_NAME.value - server_port = xds_flags.SERVER_PORT.value - server_maintenance_port = xds_flags.SERVER_MAINTENANCE_PORT.value - server_xds_host = xds_flags.SERVER_XDS_HOST.value - server_xds_port = xds_flags.SERVER_XDS_PORT.value - server_namespace = _KubernetesServerRunner.make_namespace_name( - resource_prefix, resource_suffix - ) - - gcp_api_manager = gcp.api.GcpApiManager() - - if security_mode is None: - td = traffic_director.TrafficDirectorManager( - gcp_api_manager, - project=project, - network=network, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - ) - else: - td = traffic_director.TrafficDirectorSecureManager( - gcp_api_manager, - project=project, - network=network, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - ) - if server_maintenance_port is None: - server_maintenance_port = ( - _KubernetesServerRunner.DEFAULT_SECURE_MODE_MAINTENANCE_PORT - ) - - try: - if command in ("create", "cycle"): - logger.info("Create mode") - if security_mode is None: - logger.info("No security") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - - elif security_mode == "mtls": - logger.info("Setting up mtls") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - td.setup_server_security( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - tls=True, - mtls=True, - ) - td.setup_client_security( - server_namespace=server_namespace, - server_name=server_name, - tls=True, - mtls=True, - ) - - elif security_mode == "tls": - logger.info("Setting up tls") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - td.setup_server_security( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - tls=True, - mtls=False, - ) - td.setup_client_security( - server_namespace=server_namespace, - server_name=server_name, - tls=True, - mtls=False, - ) - - elif security_mode == "plaintext": - logger.info("Setting up plaintext") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - td.setup_server_security( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - tls=False, - mtls=False, - ) - td.setup_client_security( - server_namespace=server_namespace, - server_name=server_name, - tls=False, - mtls=False, - ) - - elif security_mode == "mtls_error": - # Error case: server expects client mTLS cert, - # but client configured only for TLS - logger.info("Setting up mtls_error") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - td.setup_server_security( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - tls=True, - mtls=True, - ) - td.setup_client_security( - server_namespace=server_namespace, - server_name=server_name, - tls=True, - mtls=False, - ) - - elif security_mode == "server_authz_error": - # Error case: client does not authorize server - # because of mismatched SAN name. - logger.info("Setting up mtls_error") - td.setup_for_grpc( - server_xds_host, - server_xds_port, - health_check_port=server_maintenance_port, - ) - # Regular TLS setup, but with client policy configured using - # intentionality incorrect server_namespace. - td.setup_server_security( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - tls=True, - mtls=False, - ) - td.setup_client_security( - server_namespace=( - f"incorrect-namespace-{rand.rand_string()}" - ), - server_name=server_name, - tls=True, - mtls=False, - ) - - logger.info("Works!") - except Exception: # noqa pylint: disable=broad-except - logger.exception("Got error during creation") - - if command in ("cleanup", "cycle"): - logger.info("Cleaning up") - td.cleanup(force=True) - - if command == "backends-add": - logger.info("Adding backends") - k8s_api_manager = k8s.KubernetesApiManager( - xds_k8s_flags.KUBE_CONTEXT.value - ) - k8s_namespace = k8s.KubernetesNamespace( - k8s_api_manager, server_namespace - ) - - neg_name, neg_zones = k8s_namespace.parse_service_neg_status( - server_name, server_port - ) - - td.load_backend_service() - td.backend_service_add_neg_backends(neg_name, neg_zones) - td.wait_for_backends_healthy_status() - elif command == "backends-cleanup": - td.load_backend_service() - td.backend_service_remove_all_backends() - elif command == "unused-xds-port": - try: - unused_xds_port = td.find_unused_forwarding_rule_port() - logger.info( - "Found unused forwarding rule port: %s", unused_xds_port - ) - except Exception: # noqa pylint: disable=broad-except - logger.exception("Couldn't find unused forwarding rule port") - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py b/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py deleted file mode 100755 index ac98425247d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_test_client.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Run test xds client. - -Gamma example: -./run.sh bin/run_test_client.py --server_xds_host=psm-grpc-server \ - --server_xds_port=80 \ - --config_mesh=gketd-psm-grpc-server -""" - - -import logging -import signal - -from absl import app -from absl import flags - -from bin.lib import common -from framework import xds_flags -from framework import xds_k8s_flags -from framework.infrastructure import gcp -from framework.infrastructure import k8s - -logger = logging.getLogger(__name__) -# Flags -_CMD = flags.DEFINE_enum( - "cmd", default="run", enum_values=["run", "cleanup"], help="Command" -) -_MODE = flags.DEFINE_enum( - "mode", - default="default", - enum_values=[ - "default", - "secure", - # Uncomment if gamma-specific changes added to the client. - # "gamma", - ], - help="Select client mode", -) -_QPS = flags.DEFINE_integer("qps", default=25, help="Queries per second") -_PRINT_RESPONSE = flags.DEFINE_bool( - "print_response", default=False, help="Client prints responses" -) -_FOLLOW = flags.DEFINE_bool( - "follow", - default=False, - help=( - "Follow pod logs. Requires --collect_app_logs or" - " --debug_use_port_forwarding" - ), -) -_CONFIG_MESH = flags.DEFINE_string( - "config_mesh", - default=None, - help="Optional. Supplied to bootstrap generator to indicate AppNet mesh.", -) -_REUSE_NAMESPACE = flags.DEFINE_bool( - "reuse_namespace", default=True, help="Use existing namespace if exists" -) -_CLEANUP_NAMESPACE = flags.DEFINE_bool( - "cleanup_namespace", - default=False, - help="Delete namespace during resource cleanup", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) -# Running outside of a test suite, so require explicit resource_suffix. -flags.mark_flag_as_required(xds_flags.RESOURCE_SUFFIX.name) - - -@flags.multi_flags_validator( - (xds_flags.SERVER_XDS_PORT.name, _CMD.name), - message=( - "Run outside of a test suite, must provide" - " the exact port value (must be greater than 0)." - ), -) -def _check_server_xds_port_flag(flags_dict): - if flags_dict[_CMD.name] == "cleanup": - return True - return flags_dict[xds_flags.SERVER_XDS_PORT.name] > 0 - - -def _make_sigint_handler(client_runner: common.KubernetesClientRunner): - def sigint_handler(sig, frame): - del sig, frame - print("Caught Ctrl+C. Shutting down the logs") - client_runner.stop_pod_dependencies(log_drain_sec=3) - - return sigint_handler - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - # Log following and port forwarding. - should_follow_logs = _FOLLOW.value and xds_flags.COLLECT_APP_LOGS.value - should_port_forward = ( - should_follow_logs and xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value - ) - enable_workload_identity: bool = ( - xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value - ) - - # Setup. - gcp_api_manager = gcp.api.GcpApiManager() - k8s_api_manager = k8s.KubernetesApiManager(xds_k8s_flags.KUBE_CONTEXT.value) - client_namespace = common.make_client_namespace(k8s_api_manager) - client_runner = common.make_client_runner( - client_namespace, - gcp_api_manager, - reuse_namespace=_REUSE_NAMESPACE.value, - mode=_MODE.value, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - ) - - # Server target - server_target = f"xds:///{xds_flags.SERVER_XDS_HOST.value}" - if xds_flags.SERVER_XDS_PORT.value != 80: - server_target = f"{server_target}:{xds_flags.SERVER_XDS_PORT.value}" - - if _CMD.value == "run": - logger.info("Run client, mode=%s", _MODE.value) - client_runner.run( - server_target=server_target, - qps=_QPS.value, - print_response=_PRINT_RESPONSE.value, - secure_mode=_MODE.value == "secure", - config_mesh=_CONFIG_MESH.value, - log_to_stdout=_FOLLOW.value, - ) - if should_follow_logs: - print("Following pod logs. Press Ctrl+C top stop") - signal.signal(signal.SIGINT, _make_sigint_handler(client_runner)) - signal.pause() - - elif _CMD.value == "cleanup": - logger.info("Cleanup client") - client_runner.cleanup( - force=True, force_namespace=_CLEANUP_NAMESPACE.value - ) - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/bin/run_test_server.py b/tools/run_tests/xds_k8s_test_driver/bin/run_test_server.py deleted file mode 100755 index 6904421ebe2..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/bin/run_test_server.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Run test xds server. - -Gamma example: -./run.sh bin/run_test_server.py --mode=gamma -""" -import logging -import signal - -from absl import app -from absl import flags - -from bin.lib import common -from framework import xds_flags -from framework import xds_k8s_flags -from framework.infrastructure import gcp -from framework.infrastructure import k8s - -logger = logging.getLogger(__name__) -# Flags -_CMD = flags.DEFINE_enum( - "cmd", default="run", enum_values=["run", "cleanup"], help="Command" -) -_MODE = flags.DEFINE_enum( - "mode", - default="default", - enum_values=["default", "secure", "gamma"], - help="Select server mode", -) -_REUSE_NAMESPACE = flags.DEFINE_bool( - "reuse_namespace", default=True, help="Use existing namespace if exists" -) -_REUSE_SERVICE = flags.DEFINE_bool( - "reuse_service", default=False, help="Use existing service if exists" -) -_FOLLOW = flags.DEFINE_bool( - "follow", default=False, help="Follow pod logs. Requires --collect_app_logs" -) -_CLEANUP_NAMESPACE = flags.DEFINE_bool( - "cleanup_namespace", - default=False, - help="Delete namespace during resource cleanup", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) -# Running outside of a test suite, so require explicit resource_suffix. -flags.mark_flag_as_required("resource_suffix") - - -def _make_sigint_handler(server_runner: common.KubernetesServerRunner): - def sigint_handler(sig, frame): - del sig, frame - print("Caught Ctrl+C. Shutting down the logs") - server_runner.stop_pod_dependencies(log_drain_sec=3) - - return sigint_handler - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - should_follow_logs = _FOLLOW.value and xds_flags.COLLECT_APP_LOGS.value - should_port_forward = ( - should_follow_logs and xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value - ) - enable_workload_identity: bool = ( - xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value - ) - - # Setup. - gcp_api_manager = gcp.api.GcpApiManager() - k8s_api_manager = k8s.KubernetesApiManager(xds_k8s_flags.KUBE_CONTEXT.value) - server_namespace = common.make_server_namespace(k8s_api_manager) - - server_runner = common.make_server_runner( - server_namespace, - gcp_api_manager, - reuse_namespace=_REUSE_NAMESPACE.value, - reuse_service=_REUSE_SERVICE.value, - mode=_MODE.value, - port_forwarding=should_port_forward, - enable_workload_identity=enable_workload_identity, - ) - - if _CMD.value == "run": - logger.info("Run server, mode=%s", _MODE.value) - server_runner.run( - test_port=xds_flags.SERVER_PORT.value, - maintenance_port=xds_flags.SERVER_MAINTENANCE_PORT.value, - secure_mode=_MODE.value == "secure", - log_to_stdout=_FOLLOW.value, - ) - if should_follow_logs: - print("Following pod logs. Press Ctrl+C top stop") - signal.signal(signal.SIGINT, _make_sigint_handler(server_runner)) - signal.pause() - - elif _CMD.value == "cleanup": - logger.info("Cleanup server") - server_runner.cleanup( - force=True, force_namespace=_CLEANUP_NAMESPACE.value - ) - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/run_tests/xds_k8s_test_driver/config/common-csm.cfg b/tools/run_tests/xds_k8s_test_driver/config/common-csm.cfg deleted file mode 100644 index eca1ca9502b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/common-csm.cfg +++ /dev/null @@ -1,3 +0,0 @@ -# Common config file for PSM CSM tests. ---resource_prefix=psm-csm ---noenable_workload_identity diff --git a/tools/run_tests/xds_k8s_test_driver/config/common.cfg b/tools/run_tests/xds_k8s_test_driver/config/common.cfg deleted file mode 100644 index af8f3462434..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/common.cfg +++ /dev/null @@ -1,11 +0,0 @@ ---resource_prefix=psm-interop ---td_bootstrap_image=gcr.io/grpc-testing/td-grpc-bootstrap:7d8d90477792e2e1bfe3a3da20b3dc9ef01d326c - -# The canonical implementation of the xDS test server. -# Can be used in tests where language-specific xDS test server does not exist, -# or missing a feature required for the test. -# TODO(sergiitk): Update every ~ 6 months; next 2024-01. ---server_image_canonical=gcr.io/grpc-testing/xds-interop/java-server:canonical-v1.56 - ---logger_levels=__main__:DEBUG,framework:INFO ---verbosity=0 diff --git a/tools/run_tests/xds_k8s_test_driver/config/gamma.cfg b/tools/run_tests/xds_k8s_test_driver/config/gamma.cfg deleted file mode 100644 index 42a1b7b1e4a..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/gamma.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# Common config file for GAMMA PSM tests. -# TODO(sergiitk): delete when confirmed it's not used ---resource_prefix=psm-gamma ---noenable_workload_identity diff --git a/tools/run_tests/xds_k8s_test_driver/config/grpc-testing.cfg b/tools/run_tests/xds_k8s_test_driver/config/grpc-testing.cfg deleted file mode 100644 index 2e89c7ebca7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/grpc-testing.cfg +++ /dev/null @@ -1,9 +0,0 @@ ---flagfile=config/common.cfg ---project=grpc-testing ---network=default-vpc ---gcp_service_account=xds-k8s-interop-tests@grpc-testing.iam.gserviceaccount.com ---private_api_key_secret_name=projects/830293263384/secrets/xds-interop-tests-private-api-access-key -# Randomize xds port. ---server_xds_port=0 -# ResultStore UI doesn't support 256 colors. ---color_style=ansi16 diff --git a/tools/run_tests/xds_k8s_test_driver/config/local-dev.cfg.example b/tools/run_tests/xds_k8s_test_driver/config/local-dev.cfg.example deleted file mode 100644 index 88e3afb0138..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/local-dev.cfg.example +++ /dev/null @@ -1,62 +0,0 @@ -# Copy to local-dev.cfg; replace ${UPPERCASED_VARS}. Details in README.md. - -## Import common settings ---flagfile=config/common.cfg - -### --------------------------------- Project ---------------------------------- - -## Project settings ---project=${PROJECT_ID} ---gcp_service_account=${WORKLOAD_SA_EMAIL} ---private_api_key_secret_name=projects/${PROJECT_NUMBER}/secrets/xds-interop-tests-private-api-access-key - -### --------------------------------- Clusters --------------------------------- - -## The name of kube context to use (points to your GKE cluster). ---kube_context=${KUBE_CONTEXT} - -### ------------------------------- App images --------------------------------- - -## Test images, f.e. java v1.57.x. ---server_image=gcr.io/grpc-testing/xds-interop/java-server:v1.57.x ---client_image=gcr.io/grpc-testing/xds-interop/java-client:v1.57.x - -### ----------------------------------- App ------------------------------------ - -## Use a resource prefix to describe usage and ownership. ---resource_prefix=${USER}-psm - -## Use random port in the server xds address, f.e. xds://my-test-server:42 ---server_xds_port=0 - -## When running ./bin helpers, you might need to set randomly generated fields -## to a static value. -# --resource_suffix=dev -# --server_xds_port=1111 - -### --------------------------------- Logging ---------------------------------- - -## Verbosity: -3 (fatal/critical), -2 (error), -1 (warning), 0 (info), 1 (debug) -# --verbosity=1 - -## Uncomment and set different log levels per module. Examples: -# --logger_levels=__main__:DEBUG,framework:INFO -# --logger_levels=__main__:INFO,framework:DEBUG,urllib3.connectionpool:ERROR - -## Uncomment to collect test client, server logs to out/test_app_logs/ folder. -# --collect_app_logs -# --log_dir=out - -### ------------------------------- Local dev --------------------------------- - -## Enable port forwarding in local dev. ---debug_use_port_forwarding - -## (convenience) Allow to set always known flags. ---undefok=private_api_key_secret_name,gcp_ui_url - -## Uncomment to create the firewall rule before test case runs. -# --ensure_firewall - -## Uncomment if the health check port opened in firewall is different than 8080. -# --server_port=50051 diff --git a/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg b/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg deleted file mode 100644 index 9295d7fa53b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/config/url-map.cfg +++ /dev/null @@ -1,15 +0,0 @@ ---resource_prefix=interop-psm-url-map ---strategy=reuse ---server_xds_port=8848 - -# NOTE(lidiz) we pin the server image to java-server because: -# 1. Only Java server understands the rpc-behavior metadata. -# 2. All UrlMap tests today are testing client-side logic. -# -# TODO(sergiitk): Use --server_image_canonical instead. ---server_image=gcr.io/grpc-testing/xds-interop/java-server:canonical-v1.56 - -# Disables the GCP Workload Identity feature to simplify permission control ---gcp_service_account=None ---private_api_key_secret_name=None ---noenable_workload_identity diff --git a/tools/run_tests/xds_k8s_test_driver/framework/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/bootstrap_generator_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/bootstrap_generator_testcase.py deleted file mode 100644 index 8c26d47fdd8..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/bootstrap_generator_testcase.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Optional - -from framework import xds_k8s_testcase -from framework.helpers import rand as helpers_rand -from framework.infrastructure import k8s -from framework.infrastructure import traffic_director -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) - -# Type aliases -TrafficDirectorManager = traffic_director.TrafficDirectorManager -XdsTestServer = xds_k8s_testcase.XdsTestServer -XdsTestClient = xds_k8s_testcase.XdsTestClient -KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner -KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner - - -class BootstrapGeneratorBaseTest(xds_k8s_testcase.XdsKubernetesBaseTestCase): - """Common functionality to support testing of bootstrap generator versions - across gRPC clients and servers.""" - - @classmethod - def setUpClass(cls): - """Hook method for setting up class fixture before running tests in - the class. - """ - super().setUpClass() - if cls.server_maintenance_port is None: - cls.server_maintenance_port = ( - KubernetesServerRunner.DEFAULT_MAINTENANCE_PORT - ) - - # Bootstrap generator tests are run as parameterized tests which only - # perform steps specific to the parameterized version of the bootstrap - # generator under test. - # - # Here, we perform setup steps which are common across client and server - # side variants of the bootstrap generator test. - if cls.resource_suffix_randomize: - cls.resource_suffix = helpers_rand.random_resource_suffix() - logger.info( - "Test run resource prefix: %s, suffix: %s", - cls.resource_prefix, - cls.resource_suffix, - ) - - # TD Manager - cls.td = cls.initTrafficDirectorManager() - - # Test namespaces for client and server. - cls.server_namespace = KubernetesServerRunner.make_namespace_name( - cls.resource_prefix, cls.resource_suffix - ) - cls.client_namespace = KubernetesClientRunner.make_namespace_name( - cls.resource_prefix, cls.resource_suffix - ) - - # Ensures the firewall exist - if cls.ensure_firewall: - cls.td.create_firewall_rule( - allowed_ports=cls.firewall_allowed_ports - ) - - # Randomize xds port, when it's set to 0 - if cls.server_xds_port == 0: - # TODO(sergiitk): this is prone to race conditions: - # The port might not me taken now, but there's not guarantee - # it won't be taken until the tests get to creating - # forwarding rule. This check is better than nothing, - # but we should find a better approach. - cls.server_xds_port = cls.td.find_unused_forwarding_rule_port() - logger.info("Found unused xds port: %s", cls.server_xds_port) - - # Common TD resources across client and server tests. - cls.td.setup_for_grpc( - cls.server_xds_host, - cls.server_xds_port, - health_check_port=cls.server_maintenance_port, - ) - - @classmethod - def tearDownClass(cls): - cls.td.cleanup(force=cls.force_cleanup) - super().tearDownClass() - - @classmethod - def initTrafficDirectorManager(cls) -> TrafficDirectorManager: - return TrafficDirectorManager( - cls.gcp_api_manager, - project=cls.project, - resource_prefix=cls.resource_prefix, - resource_suffix=cls.resource_suffix, - network=cls.network, - compute_api_version=cls.compute_api_version, - ) - - @classmethod - def initKubernetesServerRunner( - cls, *, td_bootstrap_image: Optional[str] = None - ) -> KubernetesServerRunner: - if not td_bootstrap_image: - td_bootstrap_image = cls.td_bootstrap_image - return KubernetesServerRunner( - k8s.KubernetesNamespace(cls.k8s_api_manager, cls.server_namespace), - deployment_name=cls.server_name, - image_name=cls.server_image, - td_bootstrap_image=td_bootstrap_image, - gcp_project=cls.project, - gcp_api_manager=cls.gcp_api_manager, - gcp_service_account=cls.gcp_service_account, - xds_server_uri=cls.xds_server_uri, - network=cls.network, - debug_use_port_forwarding=cls.debug_use_port_forwarding, - enable_workload_identity=cls.enable_workload_identity, - ) - - @staticmethod - def startTestServer( - server_runner, - port, - maintenance_port, - xds_host, - xds_port, - replica_count=1, - **kwargs, - ) -> XdsTestServer: - test_server = server_runner.run( - replica_count=replica_count, - test_port=port, - maintenance_port=maintenance_port, - **kwargs, - )[0] - test_server.set_xds_address(xds_host, xds_port) - return test_server - - def initKubernetesClientRunner( - self, td_bootstrap_image: Optional[str] = None - ) -> KubernetesClientRunner: - if not td_bootstrap_image: - td_bootstrap_image = self.td_bootstrap_image - return KubernetesClientRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.client_namespace - ), - deployment_name=self.client_name, - image_name=self.client_image, - td_bootstrap_image=td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - stats_port=self.client_port, - reuse_namespace=self.server_namespace == self.client_namespace, - ) - - def startTestClient( - self, test_server: XdsTestServer, **kwargs - ) -> XdsTestClient: - test_client = self.client_runner.run( - server_target=test_server.xds_uri, **kwargs - ) - test_client.wait_for_server_channel_ready() - return test_client diff --git a/tools/run_tests/xds_k8s_test_driver/framework/errors.py b/tools/run_tests/xds_k8s_test_driver/framework/errors.py deleted file mode 100644 index 6b067384c9c..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/errors.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Any - - -# TODO(sergiitk): All custom error classes should extend this. -class FrameworkError(Exception): - """Base error class for framework errors.""" - - message: str - kwargs: dict[str, Any] - note: str = "" - - def __init__(self, message: str, *args, **kwargs): - self.message = message - # Exception only stores args. - self.kwargs = kwargs - # Pass to the Exception as if message is in **args. - super().__init__(*[message, *args]) - - # TODO(sergiitk): Remove in py3.11, this will be built-in. See PEP 678. - def add_note(self, note: str): - self.note = note - - def __str__(self): - return self.message if not self.note else f"{self.message}\n{self.note}" - - @classmethod - def note_blanket_error(cls, reason: str) -> str: - return f""" -Reason: {reason} -{'#' * 80} -# IMPORTANT: This is not a root cause. This is an indication that -# _something_ -- literally _anything_ -- has gone wrong in the xDS flow. -# It is _your_ responsibility to look through the interop client and/or -# server logs to determine what exactly went wrong. -{'#' * 80} -""" - - @classmethod - def note_blanket_error_info_below( - cls, reason: str, *, info_below: str - ) -> str: - return ( - f"{cls.note_blanket_error(reason)}" - f"# Please inspect the information below:\n{info_below}" - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/datetime.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/datetime.py deleted file mode 100644 index 27236d0fc06..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/datetime.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This contains common helpers for working with dates and time.""" -import datetime -import re -from typing import Optional, Pattern - -import dateutil.parser - -RE_ZERO_OFFSET: Pattern[str] = re.compile(r"[+\-]00:?00$") - - -def utc_now() -> datetime.datetime: - """Construct a datetime from current time in UTC timezone.""" - return datetime.datetime.now(datetime.timezone.utc) - - -def shorten_utc_zone(utc_datetime_str: str) -> str: - """Replace ±00:00 timezone designator with Z (zero offset AKA Zulu time).""" - return RE_ZERO_OFFSET.sub("Z", utc_datetime_str) - - -def iso8601_utc_time(time: datetime.datetime = None) -> str: - """Converts datetime UTC and formats as ISO-8601 Zulu time.""" - utc_time = time.astimezone(tz=datetime.timezone.utc) - return shorten_utc_zone(utc_time.isoformat()) - - -def iso8601_to_datetime(date_str: str) -> datetime.datetime: - # TODO(sergiitk): use regular datetime.datetime when upgraded to py3.11. - return dateutil.parser.isoparse(date_str) - - -def datetime_suffix(*, seconds: bool = False) -> str: - """Return current UTC date, and time in a format useful for resource naming. - - Examples: - - 20210626-1859 (seconds=False) - - 20210626-185942 (seconds=True) - Use in resources names incompatible with ISO 8601, e.g. some GCP resources - that only allow lowercase alphanumeric chars and dashes. - - Hours and minutes are joined together for better readability, so time is - visually distinct from dash-separated date. - """ - return utc_now().strftime("%Y%m%d-%H%M" + ("%S" if seconds else "")) - - -def ago(date_from: datetime.datetime, now: Optional[datetime.datetime] = None): - if not now: - now = utc_now() - - # Round down microseconds. - date_from = date_from.replace(microsecond=0) - now = now.replace(microsecond=0) - - # Calculate the diff. - delta: datetime.timedelta = now - date_from - - if delta.days > 1: - result = f"{delta.days} days" - elif delta.days > 0: - result = f"{delta.days} day" - else: - # This case covers negative deltas too. - result = f"{delta} (h:mm:ss)" - - return f"{result} ago" diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/grpc.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/grpc.py deleted file mode 100644 index 61cc3d6b022..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/grpc.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This contains common helpers for working with grpc data structures.""" -import dataclasses -import functools -from typing import Dict, List, Optional - -import grpc -import yaml - -from framework.rpc import grpc_testing - -# Type aliases -RpcsByPeer: Dict[str, int] -RpcMetadata = grpc_testing.LoadBalancerStatsResponse.RpcMetadata -MetadataByPeer: list[str, RpcMetadata] - - -@functools.cache # pylint: disable=no-member -def status_from_int(grpc_status_int: int) -> Optional[grpc.StatusCode]: - """Converts the integer gRPC status code to the grpc.StatusCode enum.""" - for grpc_status in grpc.StatusCode: - if grpc_status.value[0] == grpc_status_int: - return grpc_status - return None - - -def status_eq(grpc_status_int: int, grpc_status: grpc.StatusCode) -> bool: - """Compares the integer gRPC status code with the grpc.StatusCode enum.""" - return status_from_int(grpc_status_int) is grpc_status - - -def status_pretty(grpc_status: grpc.StatusCode) -> str: - """Formats the status code as (int, NAME), f.e. (4, DEADLINE_EXCEEDED)""" - return f"({grpc_status.value[0]}, {grpc_status.name})" - - -@dataclasses.dataclass(frozen=True) -class PrettyStatsPerMethod: - # The name of the method. - method: str - - # The number of RPCs started for this method, completed and in-flight. - rpcs_started: int - - # The number of RPCs that completed with each status for this method. - # Format: status code -> RPC count, f.e.: - # { - # "(0, OK)": 20, - # "(14, UNAVAILABLE)": 10 - # } - result: Dict[str, int] - - @functools.cached_property # pylint: disable=no-member - def rpcs_completed(self): - """Returns the total count of competed RPCs across all statuses.""" - return sum(self.result.values()) - - @staticmethod - def from_response( - method_name: str, method_stats: grpc_testing.MethodStats - ) -> "PrettyStatsPerMethod": - stats: Dict[str, int] = dict() - for status_int, count in method_stats.result.items(): - status: Optional[grpc.StatusCode] = status_from_int(status_int) - status_formatted = status_pretty(status) if status else "None" - stats[status_formatted] = count - return PrettyStatsPerMethod( - method=method_name, - rpcs_started=method_stats.rpcs_started, - result=stats, - ) - - -def accumulated_stats_pretty( - accumulated_stats: grpc_testing.LoadBalancerAccumulatedStatsResponse, - *, - ignore_empty: bool = False, -) -> str: - """Pretty print LoadBalancerAccumulatedStatsResponse. - - Example: - - method: EMPTY_CALL - rpcs_started: 0 - result: - (2, UNKNOWN): 20 - - method: UNARY_CALL - rpcs_started: 31 - result: - (0, OK): 10 - (14, UNAVAILABLE): 20 - """ - # Only look at stats_per_method, as the other fields are deprecated. - result: List[Dict] = [] - for method_name, method_stats in accumulated_stats.stats_per_method.items(): - pretty_stats = PrettyStatsPerMethod.from_response( - method_name, method_stats - ) - # Skip methods with no RPCs reported when ignore_empty is True. - if ignore_empty and not pretty_stats.rpcs_started: - continue - result.append(dataclasses.asdict(pretty_stats)) - - return yaml.dump(result, sort_keys=False) - - -@dataclasses.dataclass(frozen=True) -class PrettyLoadBalancerStats: - # The number of RPCs that failed to record a remote peer. - num_failures: int - - # The number of completed RPCs for each peer. - # Format: a dictionary from the host name (str) to the RPC count (int), f.e. - # {"host-a": 10, "host-b": 20} - rpcs_by_peer: "RpcsByPeer" - - # The number of completed RPCs per method per each pear. - # Format: a dictionary from the method name to RpcsByPeer (see above), f.e.: - # { - # "UNARY_CALL": {"host-a": 10, "host-b": 20}, - # "EMPTY_CALL": {"host-a": 42}, - # } - rpcs_by_method: Dict[str, "RpcsByPeer"] - - metadatas_by_peer: Dict[str, "MetadataByPeer"] - - @staticmethod - def _parse_rpcs_by_peer( - rpcs_by_peer: grpc_testing.RpcsByPeer, - ) -> "RpcsByPeer": - result = dict() - for peer, count in rpcs_by_peer.items(): - result[peer] = count - return result - - @staticmethod - def _parse_metadatas_by_peer( - metadatas_by_peer: grpc_testing.LoadBalancerStatsResponse.MetadataByPeer, - ) -> "MetadataByPeer": - result = dict() - for peer, metadatas in metadatas_by_peer.items(): - pretty_metadata = "" - for rpc_metadatas in metadatas.rpc_metadata: - for metadata in rpc_metadatas.metadata: - pretty_metadata += ( - metadata.key + ": " + metadata.value + ", " - ) - result[peer] = pretty_metadata - return result - - @classmethod - def from_response( - cls, lb_stats: grpc_testing.LoadBalancerStatsResponse - ) -> "PrettyLoadBalancerStats": - rpcs_by_method: Dict[str, "RpcsByPeer"] = dict() - for method_name, stats in lb_stats.rpcs_by_method.items(): - if stats: - rpcs_by_method[method_name] = cls._parse_rpcs_by_peer( - stats.rpcs_by_peer - ) - return PrettyLoadBalancerStats( - num_failures=lb_stats.num_failures, - rpcs_by_peer=cls._parse_rpcs_by_peer(lb_stats.rpcs_by_peer), - rpcs_by_method=rpcs_by_method, - metadatas_by_peer=cls._parse_metadatas_by_peer( - lb_stats.metadatas_by_peer - ), - ) - - -def lb_stats_pretty(lb: grpc_testing.LoadBalancerStatsResponse) -> str: - """Pretty print LoadBalancerStatsResponse. - - Example: - num_failures: 13 - rpcs_by_method: - UNARY_CALL: - psm-grpc-server-a: 100 - psm-grpc-server-b: 42 - EMPTY_CALL: - psm-grpc-server-a: 200 - rpcs_by_peer: - psm-grpc-server-a: 200 - psm-grpc-server-b: 42 - """ - pretty_lb_stats = PrettyLoadBalancerStats.from_response(lb) - stats_as_dict = dataclasses.asdict(pretty_lb_stats) - - # Don't print metadatas_by_peer unless it has data - if not stats_as_dict["metadatas_by_peer"]: - stats_as_dict.pop("metadatas_by_peer") - - return yaml.dump(stats_as_dict, sort_keys=False) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/highlighter.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/highlighter.py deleted file mode 100644 index ef6b74c72f7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/highlighter.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The module contains helpers to enable color output in terminals. - -Use this to log resources dumped as a structured document (f.e. YAML), -and enable colorful syntax highlighting. - -TODO(sergiitk): This can be used to output protobuf responses formatted as JSON. -""" -import logging -from typing import Optional - -from absl import flags -import pygments -import pygments.formatter -import pygments.formatters.other -import pygments.formatters.terminal -import pygments.formatters.terminal256 -import pygments.lexer -import pygments.lexers.data -import pygments.styles - -# The style for terminals supporting 8/16 colors. -STYLE_ANSI_16 = "ansi16" -# Join with pygments styles for terminals supporting 88/256 colors. -ALL_COLOR_STYLES = [STYLE_ANSI_16] + list(pygments.styles.get_all_styles()) - -# Flags. -COLOR = flags.DEFINE_bool("color", default=True, help="Colorize the output") -COLOR_STYLE = flags.DEFINE_enum( - "color_style", - default="material", - enum_values=ALL_COLOR_STYLES, - help=( - "Color styles for terminals supporting 256 colors. " - f"Use {STYLE_ANSI_16} style for terminals supporting 8/16 colors" - ), -) - -logger = logging.getLogger(__name__) - -# Type aliases. -Lexer = pygments.lexer.Lexer -YamlLexer = pygments.lexers.data.YamlLexer -Formatter = pygments.formatter.Formatter -NullFormatter = pygments.formatters.other.NullFormatter -TerminalFormatter = pygments.formatters.terminal.TerminalFormatter -Terminal256Formatter = pygments.formatters.terminal256.Terminal256Formatter - - -class Highlighter: - formatter: Formatter - lexer: Lexer - color: bool - color_style: Optional[str] = None - - def __init__( - self, - *, - lexer: Lexer, - color: Optional[bool] = None, - color_style: Optional[str] = None, - ): - self.lexer = lexer - self.color = color if color is not None else COLOR.value - - if self.color: - color_style = color_style if color_style else COLOR_STYLE.value - if color_style not in ALL_COLOR_STYLES: - raise ValueError( - f"Unrecognized color style {color_style}, " - f"valid styles: {ALL_COLOR_STYLES}" - ) - if color_style == STYLE_ANSI_16: - # 8/16 colors support only. - self.formatter = TerminalFormatter() - else: - # 88/256 colors. - self.formatter = Terminal256Formatter(style=color_style) - else: - self.formatter = NullFormatter() - - def highlight(self, code: str) -> str: - return pygments.highlight(code, self.lexer, self.formatter) - - -class HighlighterYaml(Highlighter): - def __init__( - self, *, color: Optional[bool] = None, color_style: Optional[str] = None - ): - super().__init__( - lexer=YamlLexer(encoding="utf-8"), - color=color, - color_style=color_style, - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/logs.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/logs.py deleted file mode 100644 index 4678737e2c2..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/logs.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The module contains helpers to initialize and configure logging.""" -import functools -import pathlib - -from absl import flags -from absl import logging - - -def _ensure_flags_parsed() -> None: - if not flags.FLAGS.is_parsed(): - raise flags.UnparsedFlagAccessError("Must initialize absl flags first.") - - -@functools.lru_cache(None) -def log_get_root_dir() -> pathlib.Path: - _ensure_flags_parsed() - log_root = pathlib.Path(logging.find_log_dir()).absolute() - logging.info("Log root dir: %s", log_root) - return log_root - - -def log_dir_mkdir(name: str) -> pathlib.Path: - """Creates and returns a subdir with the given name in the log folder.""" - if len(pathlib.Path(name).parts) != 1: - raise ValueError(f"Dir name must be a single component; got: {name}") - if ".." in name: - raise ValueError(f"Dir name must not be above the log root.") - log_subdir = log_get_root_dir() / name - if log_subdir.exists() and log_subdir.is_dir(): - logging.debug("Using existing log subdir: %s", log_subdir) - else: - log_subdir.mkdir() - logging.debug("Created log subdir: %s", log_subdir) - - return log_subdir diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py deleted file mode 100644 index 6ed79954688..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This contains common helpers for generating randomized data.""" -import random -import string - -import framework.helpers.datetime - -# Alphanumeric characters, similar to regex [:alnum:] class, [a-zA-Z0-9] -ALPHANUM = string.ascii_letters + string.digits -# Lowercase alphanumeric characters: [a-z0-9] -# Use ALPHANUM_LOWERCASE alphabet when case-sensitivity is a concern. -ALPHANUM_LOWERCASE = string.ascii_lowercase + string.digits - - -def rand_string(length: int = 8, *, lowercase: bool = False) -> str: - """Return random alphanumeric string of given length. - - Space for default arguments: alphabet^length - lowercase and uppercase = (26*2 + 10)^8 = 2.18e14 = 218 trillion. - lowercase only = (26 + 10)^8 = 2.8e12 = 2.8 trillion. - """ - alphabet = ALPHANUM_LOWERCASE if lowercase else ALPHANUM - return "".join(random.choices(population=alphabet, k=length)) - - -def random_resource_suffix() -> str: - """Return a ready-to-use resource suffix with datetime and nonce.""" - # Date and time suffix for debugging. Seconds skipped, not as relevant - # Format example: 20210626-1859 - datetime_suffix: str = framework.helpers.datetime.datetime_suffix() - # Use lowercase chars because some resource names won't allow uppercase. - # For len 5, total (26 + 10)^5 = 60,466,176 combinations. - # Approx. number of test runs needed to start at the same minute to - # produce a collision: math.sqrt(math.pi/2 * (26+10)**5) ≈ 9745. - # https://en.wikipedia.org/wiki/Birthday_attack#Mathematics - unique_hash: str = rand_string(5, lowercase=True) - return f"{datetime_suffix}-{unique_hash}" diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/retryers.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/retryers.py deleted file mode 100644 index 9f16733c3f1..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/retryers.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This contains common retrying helpers (retryers). - -We use tenacity as a general-purpose retrying library. - -> It [tenacity] originates from a fork of retrying which is sadly no -> longer maintained. Tenacity isn’t api compatible with retrying but > -> adds significant new functionality and fixes a number of longstanding bugs. -> - https://tenacity.readthedocs.io/en/latest/index.html -""" -import datetime -import logging -from typing import Any, Callable, List, Optional, Tuple, Type - -import tenacity -from tenacity import _utils as tenacity_utils -from tenacity import compat as tenacity_compat -from tenacity import stop -from tenacity import wait -from tenacity.retry import retry_base - -retryers_logger = logging.getLogger(__name__) -# Type aliases -timedelta = datetime.timedelta -Retrying = tenacity.Retrying -CheckResultFn = Callable[[Any], bool] -_ExceptionClasses = Tuple[Type[Exception], ...] - - -def _build_retry_conditions( - *, - retry_on_exceptions: Optional[_ExceptionClasses] = None, - check_result: Optional[CheckResultFn] = None, -) -> List[retry_base]: - # Retry on all exceptions by default - if retry_on_exceptions is None: - retry_on_exceptions = (Exception,) - - retry_conditions = [tenacity.retry_if_exception_type(retry_on_exceptions)] - if check_result is not None: - if retry_on_exceptions: - # When retry_on_exceptions is set, also catch them while executing - # check_result callback. - check_result = _safe_check_result(check_result, retry_on_exceptions) - retry_conditions.append(tenacity.retry_if_not_result(check_result)) - return retry_conditions - - -def exponential_retryer_with_timeout( - *, - wait_min: timedelta, - wait_max: timedelta, - timeout: timedelta, - retry_on_exceptions: Optional[_ExceptionClasses] = None, - check_result: Optional[CheckResultFn] = None, - logger: Optional[logging.Logger] = None, - log_level: Optional[int] = logging.DEBUG, -) -> Retrying: - if logger is None: - logger = retryers_logger - if log_level is None: - log_level = logging.DEBUG - - retry_conditions = _build_retry_conditions( - retry_on_exceptions=retry_on_exceptions, check_result=check_result - ) - retry_error_callback = _on_error_callback( - timeout=timeout, check_result=check_result - ) - return Retrying( - retry=tenacity.retry_any(*retry_conditions), - wait=wait.wait_exponential( - min=wait_min.total_seconds(), max=wait_max.total_seconds() - ), - stop=stop.stop_after_delay(timeout.total_seconds()), - before_sleep=_before_sleep_log(logger, log_level), - retry_error_callback=retry_error_callback, - ) - - -def constant_retryer( - *, - wait_fixed: timedelta, - attempts: int = 0, - timeout: Optional[timedelta] = None, - retry_on_exceptions: Optional[_ExceptionClasses] = None, - check_result: Optional[CheckResultFn] = None, - logger: Optional[logging.Logger] = None, - log_level: Optional[int] = logging.DEBUG, -) -> Retrying: - if logger is None: - logger = retryers_logger - if log_level is None: - log_level = logging.DEBUG - if attempts < 1 and timeout is None: - raise ValueError("The number of attempts or the timeout must be set") - stops = [] - if attempts > 0: - stops.append(stop.stop_after_attempt(attempts)) - if timeout is not None: - stops.append(stop.stop_after_delay(timeout.total_seconds())) - - retry_conditions = _build_retry_conditions( - retry_on_exceptions=retry_on_exceptions, check_result=check_result - ) - retry_error_callback = _on_error_callback( - timeout=timeout, attempts=attempts, check_result=check_result - ) - return Retrying( - retry=tenacity.retry_any(*retry_conditions), - wait=wait.wait_fixed(wait_fixed.total_seconds()), - stop=stop.stop_any(*stops), - before_sleep=_before_sleep_log(logger, log_level), - retry_error_callback=retry_error_callback, - ) - - -def _on_error_callback( - *, - timeout: Optional[timedelta] = None, - attempts: int = 0, - check_result: Optional[CheckResultFn] = None, -): - """A helper to propagate the initial state to the RetryError, so that - it can assemble a helpful message containing timeout/number of attempts. - """ - - def error_handler(retry_state: tenacity.RetryCallState): - raise RetryError( - retry_state, - timeout=timeout, - attempts=attempts, - check_result=check_result, - ) - - return error_handler - - -def _safe_check_result( - check_result: CheckResultFn, retry_on_exceptions: _ExceptionClasses -) -> CheckResultFn: - """Wraps check_result callback to catch and handle retry_on_exceptions. - - Normally tenacity doesn't retry when retry_if_result/retry_if_not_result - raise an error. This wraps the callback to automatically catch Exceptions - specified in the retry_on_exceptions argument. - - Ideally we should make all check_result callbacks to not throw, but - in case it does, we'd rather be annoying in the logs, than break the test. - """ - - def _check_result_wrapped(result): - try: - return check_result(result) - except retry_on_exceptions: - retryers_logger.warning( - ( - "Result check callback %s raised an exception." - "This shouldn't happen, please handle any exceptions and " - "return return a boolean." - ), - tenacity_utils.get_callback_name(check_result), - exc_info=True, - ) - return False - - return _check_result_wrapped - - -def _before_sleep_log(logger, log_level, exc_info=False): - """Same as tenacity.before_sleep_log, but only logs primitive return values. - This is not useful when the return value is a dump of a large object. - """ - - def log_it(retry_state): - if retry_state.outcome.failed: - ex = retry_state.outcome.exception() - verb, value = "raised", "%s: %s" % (type(ex).__name__, ex) - - if exc_info: - local_exc_info = tenacity_compat.get_exc_info_from_future( - retry_state.outcome - ) - else: - local_exc_info = False - else: - local_exc_info = False # exc_info does not apply when no exception - result = retry_state.outcome.result() - if isinstance(result, (int, bool, str)): - verb, value = "returned", result - else: - verb, value = "returned type", type(result) - - logger.log( - log_level, - "Retrying %s in %s seconds as it %s %s.", - tenacity_utils.get_callback_name(retry_state.fn), - getattr(retry_state.next_action, "sleep"), - verb, - value, - exc_info=local_exc_info, - ) - - return log_it - - -class RetryError(tenacity.RetryError): - # Note: framework.errors.FrameworkError could be used as a mixin, - # but this would rely too much on tenacity.RetryError to not change. - - last_attempt: tenacity.Future - note: str = "" - - def __init__( - self, - retry_state, - *, - timeout: Optional[timedelta] = None, - attempts: int = 0, - check_result: Optional[CheckResultFn] = None, - ): - last_attempt: tenacity.Future = retry_state.outcome - super().__init__(last_attempt) - - callback_name = tenacity_utils.get_callback_name(retry_state.fn) - self.message = f"Retry error calling {callback_name}:" - if timeout: - self.message += f" timeout {timeout} (h:mm:ss) exceeded" - if attempts: - self.message += " or" - if attempts: - self.message += f" {attempts} attempts exhausted" - - self.message += "." - - if last_attempt.failed: - err = last_attempt.exception() - self.message += f" Last exception: {type(err).__name__}: {err}" - elif check_result: - self.message += " Check result callback returned False." - - def result(self, *, default=None): - return ( - self.last_attempt.result() - if not self.last_attempt.failed - else default - ) - - def exception(self, *, default=None): - return ( - self.last_attempt.exception() - if self.last_attempt.failed - else default - ) - - # TODO(sergiitk): Remove in py3.11, this will be built-in. See PEP 678. - def add_note(self, note: str): - self.note = note - - def __str__(self): - return self.message if not self.note else f"{self.message}\n{self.note}" diff --git a/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py b/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py deleted file mode 100644 index e7c82798793..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/helpers/skips.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2022 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The classes and predicates to assist validate test config for test cases.""" -from dataclasses import dataclass -import enum -import logging -import re -from typing import Optional - -from packaging import version as pkg_version - -logger = logging.getLogger(__name__) - - -class Lang(enum.Flag): - UNKNOWN = enum.auto() - CPP = enum.auto() - GO = enum.auto() - JAVA = enum.auto() - PYTHON = enum.auto() - NODE = enum.auto() - - def __str__(self): - return str(self.name).lower() - - @classmethod - def from_string(cls, lang: str): - try: - return cls[lang.upper()] - except KeyError: - return cls.UNKNOWN - - -@dataclass -class TestConfig: - """Describes the config for the test suite. - - TODO(sergiitk): rename to LangSpec and rename skips.py to lang.py. - """ - - client_lang: Lang - server_lang: Lang - version: Optional[str] - - def version_gte(self, another: str) -> bool: - """Returns a bool for whether this VERSION is >= then ANOTHER version. - - Special cases: - - 1) Versions "master" or "dev" are always greater than ANOTHER: - - master > v1.999.x > v1.55.x - - dev > v1.999.x > v1.55.x - - dev == master - - 2) Versions "dev-VERSION" behave the same as the VERSION: - - dev-master > v1.999.x > v1.55.x - - dev-master == dev == master - - v1.55.x > dev-v1.54.x > v1.53.x - - dev-v1.54.x == v1.54.x - - 3) Unspecified version (self.version is None) is treated as "master". - """ - if self.version in ("master", "dev", "dev-master", None): - return True - - # The left side is not master, so master on the right side wins. - if another == "master": - return False - - # Treat "dev-VERSION" on the left side as "VERSION". - version: str = self.version - if version.startswith("dev-"): - version = version[4:] - return self._parse_version(version) >= self._parse_version(another) - - def __str__(self): - return ( - f"TestConfig(client_lang='{self.client_lang}', " - f"server_lang='{self.server_lang}', version={self.version!r})" - ) - - @staticmethod - def _parse_version(version: str) -> pkg_version.Version: - if version.endswith(".x"): - version = version[:-2] - return pkg_version.Version(version) - - -def get_lang(image_name: str) -> Lang: - return Lang.from_string( - re.search(r"/(\w+)-(client|server):", image_name).group(1) - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/__init__.py deleted file mode 100644 index 127ce04e92d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from framework.infrastructure.gcp import api -from framework.infrastructure.gcp import compute -from framework.infrastructure.gcp import iam -from framework.infrastructure.gcp import network_security -from framework.infrastructure.gcp import network_services diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py deleted file mode 100644 index 33e48d76009..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/api.py +++ /dev/null @@ -1,542 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import abc -import contextlib -import functools -import json -import logging -from typing import Any, Dict, List, Optional - -from absl import flags -from google.cloud import secretmanager_v1 -from google.longrunning import operations_pb2 -from google.protobuf import json_format -from google.rpc import code_pb2 -from google.rpc import error_details_pb2 -from google.rpc import status_pb2 -from googleapiclient import discovery -import googleapiclient.errors -import googleapiclient.http -import tenacity -import yaml - -import framework.helpers.highlighter - -logger = logging.getLogger(__name__) -PRIVATE_API_KEY_SECRET_NAME = flags.DEFINE_string( - "private_api_key_secret_name", - default=None, - help=( - "Load Private API access key from the latest version of the secret " - "with the given name, in the format projects/*/secrets/*" - ), -) -V1_DISCOVERY_URI = flags.DEFINE_string( - "v1_discovery_uri", - default=discovery.V1_DISCOVERY_URI, - help="Override v1 Discovery URI", -) -V2_DISCOVERY_URI = flags.DEFINE_string( - "v2_discovery_uri", - default=discovery.V2_DISCOVERY_URI, - help="Override v2 Discovery URI", -) -COMPUTE_V1_DISCOVERY_FILE = flags.DEFINE_string( - "compute_v1_discovery_file", - default=None, - help="Load compute v1 from discovery file", -) -GCP_UI_URL = flags.DEFINE_string( - "gcp_ui_url", - default="console.cloud.google.com", - help="Override GCP UI URL.", -) - -# Type aliases -_HttpError = googleapiclient.errors.HttpError -_HttpLib2Error = googleapiclient.http.httplib2.HttpLib2Error -_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml -Operation = operations_pb2.Operation -HttpRequest = googleapiclient.http.HttpRequest - - -class GcpApiManager: - def __init__( - self, - *, - v1_discovery_uri=None, - v2_discovery_uri=None, - compute_v1_discovery_file=None, - private_api_key_secret_name=None, - gcp_ui_url=None, - ): - self.v1_discovery_uri = v1_discovery_uri or V1_DISCOVERY_URI.value - self.v2_discovery_uri = v2_discovery_uri or V2_DISCOVERY_URI.value - self.compute_v1_discovery_file = ( - compute_v1_discovery_file or COMPUTE_V1_DISCOVERY_FILE.value - ) - self.private_api_key_secret_name = ( - private_api_key_secret_name or PRIVATE_API_KEY_SECRET_NAME.value - ) - self.gcp_ui_url = gcp_ui_url or GCP_UI_URL.value - # TODO(sergiitk): add options to pass google Credentials - self._exit_stack = contextlib.ExitStack() - - def close(self): - self._exit_stack.close() - - @property - @functools.lru_cache(None) - def private_api_key(self): - """ - Private API key. - - Return API key credential that identifies a GCP project allow-listed for - accessing private API discovery documents. - https://console.cloud.google.com/apis/credentials - - This method lazy-loads the content of the key from the Secret Manager. - https://console.cloud.google.com/security/secret-manager - """ - if not self.private_api_key_secret_name: - raise ValueError( - "private_api_key_secret_name must be set to " - "access private_api_key." - ) - - secrets_api = self.secrets("v1") - version_resource_path = secrets_api.secret_version_path( - **secrets_api.parse_secret_path(self.private_api_key_secret_name), - secret_version="latest", - ) - secret: secretmanager_v1.AccessSecretVersionResponse - secret = secrets_api.access_secret_version(name=version_resource_path) - return secret.payload.data.decode() - - @functools.lru_cache(None) - def compute(self, version): - api_name = "compute" - if version == "v1": - if self.compute_v1_discovery_file: - return self._build_from_file(self.compute_v1_discovery_file) - else: - return self._build_from_discovery_v1(api_name, version) - elif version == "v1alpha": - return self._build_from_discovery_v1(api_name, "alpha") - - raise NotImplementedError(f"Compute {version} not supported") - - @functools.lru_cache(None) - def networksecurity(self, version): - api_name = "networksecurity" - if version == "v1alpha1": - return self._build_from_discovery_v2( - api_name, - version, - api_key=self.private_api_key, - visibility_labels=["NETWORKSECURITY_ALPHA"], - ) - elif version == "v1beta1": - return self._build_from_discovery_v2(api_name, version) - - raise NotImplementedError(f"Network Security {version} not supported") - - @functools.lru_cache(None) - def networkservices(self, version): - api_name = "networkservices" - if version == "v1alpha1": - return self._build_from_discovery_v2( - api_name, - version, - api_key=self.private_api_key, - visibility_labels=["NETWORKSERVICES_ALPHA"], - ) - elif version == "v1beta1": - return self._build_from_discovery_v2(api_name, version) - - raise NotImplementedError(f"Network Services {version} not supported") - - @staticmethod - @functools.lru_cache(None) - def secrets(version: str): - if version == "v1": - return secretmanager_v1.SecretManagerServiceClient() - - raise NotImplementedError(f"Secret Manager {version} not supported") - - @functools.lru_cache(None) - def iam(self, version: str) -> discovery.Resource: - """Identity and Access Management (IAM) API. - - https://cloud.google.com/iam/docs/reference/rest - https://googleapis.github.io/google-api-python-client/docs/dyn/iam_v1.html - """ - api_name = "iam" - if version == "v1": - return self._build_from_discovery_v1(api_name, version) - - raise NotImplementedError( - f"Identity and Access Management (IAM) {version} not supported" - ) - - def _build_from_discovery_v1(self, api_name, version): - api = discovery.build( - api_name, - version, - cache_discovery=False, - discoveryServiceUrl=self.v1_discovery_uri, - ) - self._exit_stack.enter_context(api) - return api - - def _build_from_discovery_v2( - self, - api_name, - version, - *, - api_key: Optional[str] = None, - visibility_labels: Optional[List] = None, - ): - params = {} - if api_key: - params["key"] = api_key - if visibility_labels: - # Dash-separated list of labels. - params["labels"] = "_".join(visibility_labels) - - params_str = "" - if params: - params_str = "&" + "&".join(f"{k}={v}" for k, v in params.items()) - - api = discovery.build( - api_name, - version, - cache_discovery=False, - discoveryServiceUrl=f"{self.v2_discovery_uri}{params_str}", - ) - self._exit_stack.enter_context(api) - return api - - def _build_from_file(self, discovery_file): - with open(discovery_file, "r") as f: - api = discovery.build_from_document(f.read()) - self._exit_stack.enter_context(api) - return api - - -class Error(Exception): - """Base error class for GCP API errors.""" - - -class ResponseError(Error): - """The response was not a 2xx.""" - - reason: str - uri: str - error_details: Optional[str] - status: Optional[int] - cause: _HttpError - - def __init__(self, cause: _HttpError): - # TODO(sergiitk): cleanup when we upgrade googleapiclient: - # - remove _get_reason() - # - remove error_details note - # - use status_code() - self.reason = cause._get_reason().strip() # noqa - self.uri = cause.uri - self.error_details = cause.error_details # NOTE: Must after _get_reason - self.status = None - if cause.resp and cause.resp.status: - self.status = cause.resp.status - self.cause = cause - super().__init__() - - def __repr__(self): - return ( - f"' - ) - - -class TransportError(Error): - """A transport error has occurred.""" - - cause: _HttpLib2Error - - def __init__(self, cause: _HttpLib2Error): - self.cause = cause - super().__init__() - - def __repr__(self): - return f"" - - -class OperationError(Error): - """ - Operation was not successful. - - Assuming Operation based on Google API Style Guide: - https://cloud.google.com/apis/design/design_patterns#long_running_operations - https://github.com/googleapis/googleapis/blob/master/google/longrunning/operations.proto - """ - - api_name: str - name: str - metadata: Any - code_name: code_pb2.Code - error: status_pb2.Status - - def __init__(self, api_name: str, response: dict): - self.api_name = api_name - - # Operation.metadata field is Any specific to the API. It may not be - # present in the default descriptor pool, and that's expected. - # To avoid json_format.ParseError, handle it separately. - self.metadata = response.pop("metadata", {}) - - # Must be after removing metadata field. - operation: Operation = self._parse_operation_response(response) - self.name = operation.name or "unknown" - self.code_name = code_pb2.Code.Name(operation.error.code) - self.error = operation.error - super().__init__() - - @staticmethod - def _parse_operation_response(operation_response: dict) -> Operation: - try: - return json_format.ParseDict( - operation_response, - Operation(), - ignore_unknown_fields=True, - descriptor_pool=error_details_pb2.DESCRIPTOR.pool, - ) - except (json_format.Error, TypeError) as e: - # Swallow parsing errors if any. Building correct OperationError() - # is more important than losing debug information. Details still - # can be extracted from the warning. - logger.warning( - ( - "Can't parse response while processing OperationError:" - " '%r', error %r" - ), - operation_response, - e, - ) - return Operation() - - def __str__(self): - indent_l1 = " " * 2 - indent_l2 = indent_l1 * 2 - - result = ( - f'{self.api_name} operation "{self.name}" failed.\n' - f"{indent_l1}code: {self.error.code} ({self.code_name})\n" - f'{indent_l1}message: "{self.error.message}"' - ) - - if self.error.details: - result += f"\n{indent_l1}details: [\n" - for any_error in self.error.details: - error_str = json_format.MessageToJson(any_error) - for line in error_str.splitlines(): - result += indent_l2 + line + "\n" - result += f"{indent_l1}]" - - if self.metadata: - result += f"\n metadata: \n" - metadata_str = json.dumps(self.metadata, indent=2) - for line in metadata_str.splitlines(): - result += indent_l2 + line + "\n" - result = result.rstrip() - - return result - - -class GcpProjectApiResource: - # TODO(sergiitk): move someplace better - _WAIT_FOR_OPERATION_SEC = 60 * 10 - _WAIT_FIXED_SEC = 2 - _GCP_API_RETRIES = 5 - - def __init__(self, api: discovery.Resource, project: str): - self.api: discovery.Resource = api - self.project: str = project - self._highlighter = _HighlighterYaml() - - # TODO(sergiitk): in upcoming GCP refactoring, differentiate between - # _execute for LRO (Long Running Operations), and immediate operations. - def _execute( - self, - request: HttpRequest, - *, - num_retries: Optional[int] = _GCP_API_RETRIES, - ) -> Dict[str, Any]: - """Execute the immediate request. - - Returns: - Unmarshalled response as a dictionary. - - Raises: - ResponseError if the response was not a 2xx. - TransportError if a transport error has occurred. - """ - if num_retries is None: - num_retries = self._GCP_API_RETRIES - try: - return request.execute(num_retries=num_retries) - except _HttpError as error: - raise ResponseError(error) - except _HttpLib2Error as error: - raise TransportError(error) - - def resource_pretty_format( - self, - resource: Any, - *, - highlight: bool = True, - ) -> str: - """Return a string with pretty-printed resource body.""" - yaml_out: str = yaml.dump( - resource, - explicit_start=True, - explicit_end=True, - ) - return self._highlighter.highlight(yaml_out) if highlight else yaml_out - - def resources_pretty_format( - self, - resources: list[Any], - *, - highlight: bool = True, - ) -> str: - out = [] - for resource in resources: - if hasattr(resource, "name"): - out.append(f"{resource.name}:") - elif "name" in resource: - out.append(f"{resource['name']}:") - out.append( - self.resource_pretty_format(resource, highlight=highlight) - ) - return "\n".join(out) - - @staticmethod - def wait_for_operation( - operation_request, - test_success_fn, - timeout_sec=_WAIT_FOR_OPERATION_SEC, - wait_sec=_WAIT_FIXED_SEC, - ): - retryer = tenacity.Retrying( - retry=( - tenacity.retry_if_not_result(test_success_fn) - | tenacity.retry_if_exception_type() - ), - wait=tenacity.wait_fixed(wait_sec), - stop=tenacity.stop_after_delay(timeout_sec), - after=tenacity.after_log(logger, logging.DEBUG), - reraise=True, - ) - return retryer(operation_request.execute) - - -class GcpStandardCloudApiResource(GcpProjectApiResource, metaclass=abc.ABCMeta): - GLOBAL_LOCATION = "global" - - def parent(self, location: Optional[str] = GLOBAL_LOCATION): - if location is None: - location = self.GLOBAL_LOCATION - return f"projects/{self.project}/locations/{location}" - - def resource_full_name(self, name, collection_name): - return f"{self.parent()}/{collection_name}/{name}" - - def _create_resource( - self, collection: discovery.Resource, body: dict, **kwargs - ): - logger.info( - "Creating %s resource:\n%s", - self.api_name, - self.resource_pretty_format(body), - ) - create_req = collection.create( - parent=self.parent(), body=body, **kwargs - ) - self._execute(create_req) - - @property - @abc.abstractmethod - def api_name(self) -> str: - raise NotImplementedError - - @property - @abc.abstractmethod - def api_version(self) -> str: - raise NotImplementedError - - def _get_resource(self, collection: discovery.Resource, full_name): - resource = collection.get(name=full_name).execute() - logger.info( - "Loaded %s:\n%s", full_name, self.resource_pretty_format(resource) - ) - return resource - - def _delete_resource( - self, collection: discovery.Resource, full_name: str - ) -> bool: - logger.debug("Deleting %s", full_name) - try: - self._execute(collection.delete(name=full_name)) - return True - except _HttpError as error: - if error.resp and error.resp.status == 404: - logger.debug("%s not deleted since it doesn't exist", full_name) - else: - logger.warning("Failed to delete %s, %r", full_name, error) - return False - - # TODO(sergiitk): Use ResponseError and TransportError - def _execute( # pylint: disable=arguments-differ - self, - request: HttpRequest, - timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, - ): - operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.debug("Operation %s", operation) - self._wait(operation["name"], timeout_sec) - - def _wait( - self, - operation_id: str, - timeout_sec: int = GcpProjectApiResource._WAIT_FOR_OPERATION_SEC, - ): - logger.info( - "Waiting %s sec for %s operation id: %s", - timeout_sec, - self.api_name, - operation_id, - ) - - op_request = ( - self.api.projects().locations().operations().get(name=operation_id) - ) - operation = self.wait_for_operation( - operation_request=op_request, - test_success_fn=lambda result: result["done"], - timeout_sec=timeout_sec, - ) - - logger.debug("Completed operation: %s", operation) - if "error" in operation: - raise OperationError(self.api_name, operation) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py deleted file mode 100644 index b06a1007b4e..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/compute.py +++ /dev/null @@ -1,637 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import dataclasses -import datetime -import enum -import logging -from typing import Any, Dict, List, Optional, Set - -from googleapiclient import discovery -import googleapiclient.errors -import httplib2 - -import framework.errors -from framework.helpers import retryers -from framework.infrastructure import gcp - -logger = logging.getLogger(__name__) - -DEBUG_HEADER_IN_RESPONSE = "x-encrypted-debug-headers" -DEBUG_HEADER_KEY = "X-Return-Encrypted-Headers" - - -class ComputeV1( - gcp.api.GcpProjectApiResource -): # pylint: disable=too-many-public-methods - # TODO(sergiitk): move someplace better - _WAIT_FOR_BACKEND_SEC = 60 * 10 - _WAIT_FOR_BACKEND_SLEEP_SEC = 4 - _WAIT_FOR_OPERATION_SEC = 60 * 10 - gfe_debug_header: Optional[str] - - @dataclasses.dataclass(frozen=True) - class GcpResource: - name: str - url: str - - @dataclasses.dataclass(frozen=True) - class ZonalGcpResource(GcpResource): - zone: str - - def __init__( - self, - api_manager: gcp.api.GcpApiManager, - project: str, - gfe_debug_header: Optional[str] = None, - version: str = "v1", - ): - super().__init__(api_manager.compute(version), project) - self.gfe_debug_header = gfe_debug_header - - class HealthCheckProtocol(enum.Enum): - TCP = enum.auto() - GRPC = enum.auto() - - class BackendServiceProtocol(enum.Enum): - HTTP2 = enum.auto() - GRPC = enum.auto() - - def create_health_check( - self, - name: str, - protocol: HealthCheckProtocol, - *, - port: Optional[int] = None, - ) -> "GcpResource": - if protocol is self.HealthCheckProtocol.TCP: - health_check_field = "tcpHealthCheck" - elif protocol is self.HealthCheckProtocol.GRPC: - health_check_field = "grpcHealthCheck" - else: - raise TypeError(f"Unexpected Health Check protocol: {protocol}") - - health_check_settings = {} - if port is None: - health_check_settings["portSpecification"] = "USE_SERVING_PORT" - else: - health_check_settings["portSpecification"] = "USE_FIXED_PORT" - health_check_settings["port"] = port - - return self._insert_resource( - self.api.healthChecks(), - { - "name": name, - "type": protocol.name, - health_check_field: health_check_settings, - }, - ) - - def list_health_check(self): - return self._list_resource(self.api.healthChecks()) - - def delete_health_check(self, name): - self._delete_resource(self.api.healthChecks(), "healthCheck", name) - - def create_firewall_rule( - self, - name: str, - network_url: str, - source_ranges: List[str], - ports: List[str], - ) -> Optional["GcpResource"]: - try: - return self._insert_resource( - self.api.firewalls(), - { - "allowed": [{"IPProtocol": "tcp", "ports": ports}], - "direction": "INGRESS", - "name": name, - "network": network_url, - "priority": 1000, - "sourceRanges": source_ranges, - "targetTags": ["allow-health-checks"], - }, - ) - except googleapiclient.errors.HttpError as http_error: - # TODO(lidiz) use status_code() when we upgrade googleapiclient - if http_error.resp.status == 409: - logger.debug("Firewall rule %s already existed", name) - return None - else: - raise - - def delete_firewall_rule(self, name): - self._delete_resource(self.api.firewalls(), "firewall", name) - - def create_backend_service_traffic_director( - self, - name: str, - health_check: "GcpResource", - affinity_header: Optional[str] = None, - protocol: Optional[BackendServiceProtocol] = None, - subset_size: Optional[int] = None, - locality_lb_policies: Optional[List[dict]] = None, - outlier_detection: Optional[dict] = None, - ) -> "GcpResource": - if not isinstance(protocol, self.BackendServiceProtocol): - raise TypeError(f"Unexpected Backend Service protocol: {protocol}") - body = { - "name": name, - "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director - "healthChecks": [health_check.url], - "protocol": protocol.name, - } - # If affinity header is specified, config the backend service to support - # affinity, and set affinity header to the one given. - if affinity_header: - body["sessionAffinity"] = "HEADER_FIELD" - body["localityLbPolicy"] = "RING_HASH" - body["consistentHash"] = { - "httpHeaderName": affinity_header, - } - if subset_size: - body["subsetting"] = { - "policy": "CONSISTENT_HASH_SUBSETTING", - "subsetSize": subset_size, - } - if locality_lb_policies: - body["localityLbPolicies"] = locality_lb_policies - if outlier_detection: - body["outlierDetection"] = outlier_detection - return self._insert_resource(self.api.backendServices(), body) - - def get_backend_service_traffic_director(self, name: str) -> "GcpResource": - return self._get_resource( - self.api.backendServices(), backendService=name - ) - - def patch_backend_service(self, backend_service, body, **kwargs): - self._patch_resource( - collection=self.api.backendServices(), - backendService=backend_service.name, - body=body, - **kwargs, - ) - - def backend_service_patch_backends( - self, - backend_service, - backends, - max_rate_per_endpoint: Optional[int] = None, - ): - if max_rate_per_endpoint is None: - max_rate_per_endpoint = 5 - backend_list = [ - { - "group": backend.url, - "balancingMode": "RATE", - "maxRatePerEndpoint": max_rate_per_endpoint, - } - for backend in backends - ] - - self._patch_resource( - collection=self.api.backendServices(), - body={"backends": backend_list}, - backendService=backend_service.name, - ) - - def backend_service_remove_all_backends(self, backend_service): - self._patch_resource( - collection=self.api.backendServices(), - body={"backends": []}, - backendService=backend_service.name, - ) - - def delete_backend_service(self, name): - self._delete_resource( - self.api.backendServices(), "backendService", name - ) - - def create_url_map( - self, - name: str, - matcher_name: str, - src_hosts, - dst_default_backend_service: "GcpResource", - dst_host_rule_match_backend_service: Optional["GcpResource"] = None, - ) -> "GcpResource": - if dst_host_rule_match_backend_service is None: - dst_host_rule_match_backend_service = dst_default_backend_service - return self._insert_resource( - self.api.urlMaps(), - { - "name": name, - "defaultService": dst_default_backend_service.url, - "hostRules": [ - { - "hosts": src_hosts, - "pathMatcher": matcher_name, - } - ], - "pathMatchers": [ - { - "name": matcher_name, - "defaultService": dst_host_rule_match_backend_service.url, - } - ], - }, - ) - - def create_url_map_with_content(self, url_map_body: Any) -> "GcpResource": - return self._insert_resource(self.api.urlMaps(), url_map_body) - - def patch_url_map(self, url_map: "GcpResource", body, **kwargs): - self._patch_resource( - collection=self.api.urlMaps(), - urlMap=url_map.name, - body=body, - **kwargs, - ) - - def delete_url_map(self, name): - self._delete_resource(self.api.urlMaps(), "urlMap", name) - - def create_target_grpc_proxy( - self, - name: str, - url_map: "GcpResource", - validate_for_proxyless: bool = True, - ) -> "GcpResource": - return self._insert_resource( - self.api.targetGrpcProxies(), - { - "name": name, - "url_map": url_map.url, - "validate_for_proxyless": validate_for_proxyless, - }, - ) - - def delete_target_grpc_proxy(self, name): - self._delete_resource( - self.api.targetGrpcProxies(), "targetGrpcProxy", name - ) - - def create_target_http_proxy( - self, - name: str, - url_map: "GcpResource", - ) -> "GcpResource": - return self._insert_resource( - self.api.targetHttpProxies(), - { - "name": name, - "url_map": url_map.url, - }, - ) - - def delete_target_http_proxy(self, name): - self._delete_resource( - self.api.targetHttpProxies(), "targetHttpProxy", name - ) - - def create_forwarding_rule( - self, - name: str, - src_port: int, - target_proxy: "GcpResource", - network_url: str, - *, - ip_address: str = "0.0.0.0", - ) -> "GcpResource": - return self._insert_resource( - self.api.globalForwardingRules(), - { - "name": name, - "loadBalancingScheme": "INTERNAL_SELF_MANAGED", # Traffic Director - "portRange": src_port, - "IPAddress": ip_address, - "network": network_url, - "target": target_proxy.url, - }, - ) - - def exists_forwarding_rule(self, src_port) -> bool: - # TODO(sergiitk): Better approach for confirming the port is available. - # It's possible a rule allocates actual port range, e.g 8000-9000, - # and this wouldn't catch it. For now, we assume there's no - # port ranges used in the project. - filter_str = ( - f'(portRange eq "{src_port}-{src_port}") ' - '(IPAddress eq "0.0.0.0")' - '(loadBalancingScheme eq "INTERNAL_SELF_MANAGED")' - ) - return self._exists_resource( - self.api.globalForwardingRules(), resource_filter=filter_str - ) - - def delete_forwarding_rule(self, name): - self._delete_resource( - self.api.globalForwardingRules(), "forwardingRule", name - ) - - def wait_for_network_endpoint_group( - self, - name: str, - zone: str, - *, - timeout_sec=_WAIT_FOR_BACKEND_SEC, - wait_sec=_WAIT_FOR_BACKEND_SLEEP_SEC, - ): - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta(seconds=wait_sec), - timeout=datetime.timedelta(seconds=timeout_sec), - check_result=lambda neg: neg and neg.get("size", 0) > 0, - ) - network_endpoint_group = retryer( - self._retry_load_network_endpoint_group, name, zone - ) - # TODO(sergiitk): dataclass - return self.ZonalGcpResource( - network_endpoint_group["name"], - network_endpoint_group["selfLink"], - zone, - ) - - def _retry_load_network_endpoint_group(self, name: str, zone: str): - try: - neg = self.get_network_endpoint_group(name, zone) - logger.debug( - "Waiting for endpoints: NEG %s in zone %s, current count %s", - neg["name"], - zone, - neg.get("size"), - ) - except googleapiclient.errors.HttpError as error: - # noinspection PyProtectedMember - reason = error._get_reason() - logger.debug( - "Retrying NEG load, got %s, details %s", - error.resp.status, - reason, - ) - raise - return neg - - def get_network_endpoint_group(self, name, zone): - neg = ( - self.api.networkEndpointGroups() - .get(project=self.project, networkEndpointGroup=name, zone=zone) - .execute() - ) - # TODO(sergiitk): dataclass - return neg - - def wait_for_backends_healthy_status( - self, - backend_service: GcpResource, - backends: Set[ZonalGcpResource], - *, - timeout_sec: int = _WAIT_FOR_BACKEND_SEC, - wait_sec: int = _WAIT_FOR_BACKEND_SLEEP_SEC, - ) -> None: - if not backends: - raise ValueError("The list of backends to wait on is empty") - - timeout = datetime.timedelta(seconds=timeout_sec) - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta(seconds=wait_sec), - timeout=timeout, - check_result=lambda result: result, - ) - pending = set(backends) - try: - retryer(self._retry_backends_health, backend_service, pending) - except retryers.RetryError as retry_err: - unhealthy_backends: str = ",".join( - [backend.name for backend in pending] - ) - - # Attempt to load backend health info for better debug info. - try: - unhealthy = [] - # Everything left in pending was unhealthy on the last retry. - for backend in pending: - # It's possible the health status has changed since we - # gave up retrying, but this should be very rare. - health_status = self.get_backend_service_backend_health( - backend_service, - backend, - ) - unhealthy.append( - {"name": backend.name, "health_status": health_status} - ) - - # Override the plain list of unhealthy backend name with - # the one showing the latest backend statuses. - unhealthy_backends = self.resources_pretty_format( - unhealthy, - highlight=False, - ) - except Exception as error: # noqa pylint: disable=broad-except - logger.debug( - "Couldn't load backend health info, plain list name" - "will be printed instead. Error: %r", - error, - ) - - retry_err.add_note( - framework.errors.FrameworkError.note_blanket_error_info_below( - "One or several NEGs (Network Endpoint Groups) didn't" - " report HEALTHY status within expected timeout.", - info_below=( - f"Timeout {timeout} (h:mm:ss) waiting for backend" - f" service '{backend_service.name}' to report all NEGs" - " in the HEALTHY status:" - f" {[backend.name for backend in backends]}." - f"\nUnhealthy backends:\n{unhealthy_backends}" - ), - ) - ) - - raise - - def _retry_backends_health( - self, backend_service: GcpResource, pending: Set[ZonalGcpResource] - ): - for backend in pending: - result = self.get_backend_service_backend_health( - backend_service, backend - ) - if "healthStatus" not in result: - logger.debug( - "Waiting for instances: backend %s, zone %s", - backend.name, - backend.zone, - ) - continue - - backend_healthy = True - for instance in result["healthStatus"]: - logger.debug( - "Backend %s in zone %s: instance %s:%s health: %s", - backend.name, - backend.zone, - instance["ipAddress"], - instance["port"], - instance["healthState"], - ) - if instance["healthState"] != "HEALTHY": - backend_healthy = False - - if backend_healthy: - logger.info( - "Backend %s in zone %s reported healthy", - backend.name, - backend.zone, - ) - pending.remove(backend) - - return not pending - - def get_backend_service_backend_health(self, backend_service, backend): - return ( - self.api.backendServices() - .getHealth( - project=self.project, - backendService=backend_service.name, - body={"group": backend.url}, - ) - .execute() - ) - - def _get_resource( - self, collection: discovery.Resource, **kwargs - ) -> "GcpResource": - resp = collection.get(project=self.project, **kwargs).execute() - logger.info( - "Loaded compute resource:\n%s", self.resource_pretty_format(resp) - ) - return self.GcpResource(resp["name"], resp["selfLink"]) - - def _exists_resource( - self, collection: discovery.Resource, resource_filter: str - ) -> bool: - resp = collection.list( - project=self.project, filter=resource_filter, maxResults=1 - ).execute(num_retries=self._GCP_API_RETRIES) - if "kind" not in resp: - # TODO(sergiitk): better error - raise ValueError('List response "kind" is missing') - return "items" in resp and resp["items"] - - def _insert_resource( - self, collection: discovery.Resource, body: Dict[str, Any] - ) -> "GcpResource": - logger.info( - "Creating compute resource:\n%s", self.resource_pretty_format(body) - ) - resp = self._execute(collection.insert(project=self.project, body=body)) - return self.GcpResource(body["name"], resp["targetLink"]) - - def _patch_resource(self, collection, body, **kwargs): - logger.info( - "Patching compute resource:\n%s", self.resource_pretty_format(body) - ) - self._execute( - collection.patch(project=self.project, body=body, **kwargs) - ) - - def _list_resource(self, collection: discovery.Resource): - return collection.list(project=self.project).execute( - num_retries=self._GCP_API_RETRIES - ) - - def _delete_resource( - self, - collection: discovery.Resource, - resource_type: str, - resource_name: str, - ) -> bool: - try: - params = {"project": self.project, resource_type: resource_name} - self._execute(collection.delete(**params)) - return True - except googleapiclient.errors.HttpError as error: - if error.resp and error.resp.status == 404: - logger.debug( - "Resource %s %s not deleted since it doesn't exist", - resource_type, - resource_name, - ) - else: - logger.warning( - 'Failed to delete %s "%s", %r', - resource_type, - resource_name, - error, - ) - return False - - @staticmethod - def _operation_status_done(operation): - return "status" in operation and operation["status"] == "DONE" - - @staticmethod - def _log_debug_header(resp: httplib2.Response): - if ( - DEBUG_HEADER_IN_RESPONSE in resp - and resp.status >= 300 - and resp.status != 404 - ): - logger.info( - "Received GCP debug headers: %s", - resp[DEBUG_HEADER_IN_RESPONSE], - ) - - def _execute( # pylint: disable=arguments-differ - self, request, *, timeout_sec=_WAIT_FOR_OPERATION_SEC - ): - if self.gfe_debug_header: - logger.debug( - "Adding debug headers for method: %s", request.methodId - ) - request.headers[DEBUG_HEADER_KEY] = self.gfe_debug_header - request.add_response_callback(self._log_debug_header) - operation = request.execute(num_retries=self._GCP_API_RETRIES) - logger.debug("Operation %s", operation) - return self._wait(operation["name"], timeout_sec) - - def _wait( - self, operation_id: str, timeout_sec: int = _WAIT_FOR_OPERATION_SEC - ) -> dict: - logger.info( - "Waiting %s sec for compute operation id: %s", - timeout_sec, - operation_id, - ) - - # TODO(sergiitk) try using wait() here - # https://googleapis.github.io/google-api-python-client/docs/dyn/compute_v1.globalOperations.html#wait - op_request = self.api.globalOperations().get( - project=self.project, operation=operation_id - ) - operation = self.wait_for_operation( - operation_request=op_request, - test_success_fn=self._operation_status_done, - timeout_sec=timeout_sec, - ) - - logger.debug("Completed operation: %s", operation) - if "error" in operation: - # This shouldn't normally happen: gcp library raises on errors. - raise Exception( - f"Compute operation {operation_id} failed: {operation}" - ) - return operation diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/iam.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/iam.py deleted file mode 100644 index fc153da08d7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/iam.py +++ /dev/null @@ -1,361 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import dataclasses -import datetime -import functools -import logging -from typing import Any, Dict, FrozenSet, Optional - -from framework.helpers import retryers -from framework.infrastructure import gcp - -logger = logging.getLogger(__name__) - -# Type aliases -_timedelta = datetime.timedelta -_HttpRequest = gcp.api.HttpRequest - - -class EtagConflict(gcp.api.Error): - """ - Indicates concurrent policy changes. - - https://cloud.google.com/iam/docs/policies#etag - """ - - -def handle_etag_conflict(func): - def wrap_retry_on_etag_conflict(*args, **kwargs): - retryer = retryers.exponential_retryer_with_timeout( - retry_on_exceptions=(EtagConflict, gcp.api.TransportError), - wait_min=_timedelta(seconds=1), - wait_max=_timedelta(seconds=10), - timeout=_timedelta(minutes=2), - ) - return retryer(func, *args, **kwargs) - - return wrap_retry_on_etag_conflict - - -def _replace_binding( - policy: "Policy", binding: "Policy.Binding", new_binding: "Policy.Binding" -) -> "Policy": - new_bindings = set(policy.bindings) - new_bindings.discard(binding) - new_bindings.add(new_binding) - # pylint: disable=too-many-function-args # No idea why pylint is like that. - return dataclasses.replace(policy, bindings=frozenset(new_bindings)) - - -@dataclasses.dataclass(frozen=True) -class ServiceAccount: - """An IAM service account. - - https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts - Note: "etag" field is skipped because it's deprecated - """ - - name: str - projectId: str - uniqueId: str - email: str - oauth2ClientId: str - displayName: str = "" - description: str = "" - disabled: bool = False - - @classmethod - def from_response(cls, response: Dict[str, Any]) -> "ServiceAccount": - return cls( - name=response["name"], - projectId=response["projectId"], - uniqueId=response["uniqueId"], - email=response["email"], - oauth2ClientId=response["oauth2ClientId"], - description=response.get("description", ""), - displayName=response.get("displayName", ""), - disabled=response.get("disabled", False), - ) - - def as_dict(self) -> Dict[str, Any]: - return dataclasses.asdict(self) - - -@dataclasses.dataclass(frozen=True) -class Expr: - """ - Represents a textual expression in the Common Expression Language syntax. - - https://cloud.google.com/iam/docs/reference/rest/v1/Expr - """ - - expression: str - title: str = "" - description: str = "" - location: str = "" - - @classmethod - def from_response(cls, response: Dict[str, Any]) -> "Expr": - return cls(**response) - - def as_dict(self) -> Dict[str, Any]: - return dataclasses.asdict(self) - - -@dataclasses.dataclass(frozen=True) -class Policy: - """An Identity and Access Management (IAM) policy, which specifies - access controls for Google Cloud resources. - - https://cloud.google.com/iam/docs/reference/rest/v1/Policy - Note: auditConfigs not supported by this implementation. - """ - - @dataclasses.dataclass(frozen=True) - class Binding: - """Policy Binding. Associates members with a role. - - https://cloud.google.com/iam/docs/reference/rest/v1/Policy#binding - """ - - role: str - members: FrozenSet[str] - condition: Optional[Expr] = None - - @classmethod - def from_response(cls, response: Dict[str, Any]) -> "Policy.Binding": - fields = { - "role": response["role"], - "members": frozenset(response.get("members", [])), - } - if "condition" in response: - fields["condition"] = Expr.from_response(response["condition"]) - - return cls(**fields) - - def as_dict(self) -> Dict[str, Any]: - result = { - "role": self.role, - "members": list(self.members), - } - if self.condition is not None: - result["condition"] = self.condition.as_dict() - return result - - bindings: FrozenSet[Binding] - etag: str - version: Optional[int] = None - - @functools.lru_cache(maxsize=128) - def find_binding_for_role( - self, role: str, condition: Optional[Expr] = None - ) -> Optional["Policy.Binding"]: - results = ( - binding - for binding in self.bindings - if binding.role == role and binding.condition == condition - ) - return next(results, None) - - @classmethod - def from_response(cls, response: Dict[str, Any]) -> "Policy": - bindings = frozenset( - cls.Binding.from_response(b) for b in response.get("bindings", []) - ) - return cls( - bindings=bindings, - etag=response["etag"], - version=response.get("version"), - ) - - def as_dict(self) -> Dict[str, Any]: - result = { - "bindings": [binding.as_dict() for binding in self.bindings], - "etag": self.etag, - } - if self.version is not None: - result["version"] = self.version - return result - - -class IamV1(gcp.api.GcpProjectApiResource): - """ - Identity and Access Management (IAM) API. - - https://cloud.google.com/iam/docs/reference/rest - """ - - _service_accounts: gcp.api.discovery.Resource - - # Operations that affect conditional role bindings must specify version 3. - # Otherwise conditions are omitted, and role names returned with a suffix, - # f.e. roles/iam.workloadIdentityUser_withcond_f1ec33c9beb41857dbf0 - # https://cloud.google.com/iam/docs/reference/rest/v1/Policy#FIELDS.version - POLICY_VERSION: int = 3 - - def __init__(self, api_manager: gcp.api.GcpApiManager, project: str): - super().__init__(api_manager.iam("v1"), project) - # Shortcut to projects/*/serviceAccounts/ endpoints - self._service_accounts = self.api.projects().serviceAccounts() - - def service_account_resource_name(self, account) -> str: - """ - Returns full resource name of the service account. - - The resource name of the service account in the following format: - projects/{PROJECT_ID}/serviceAccounts/{ACCOUNT}. - The ACCOUNT value can be the email address or the uniqueId of the - service account. - Ref https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts/get - - Args: - account: The ACCOUNT value - """ - return f"projects/{self.project}/serviceAccounts/{account}" - - def get_service_account(self, account: str) -> ServiceAccount: - resource_name = self.service_account_resource_name(account) - request: _HttpRequest = self._service_accounts.get(name=resource_name) - response: Dict[str, Any] = self._execute(request) - logger.debug( - "Loaded Service Account:\n%s", self.resource_pretty_format(response) - ) - return ServiceAccount.from_response(response) - - def get_service_account_iam_policy(self, account: str) -> Policy: - resource_name = self.service_account_resource_name(account) - request: _HttpRequest = self._service_accounts.getIamPolicy( - resource=resource_name, - options_requestedPolicyVersion=self.POLICY_VERSION, - ) - response: Dict[str, Any] = self._execute(request) - logger.debug( - "Loaded Service Account Policy:\n%s", - self.resource_pretty_format(response), - ) - return Policy.from_response(response) - - def set_service_account_iam_policy( - self, account: str, policy: Policy - ) -> Policy: - """Sets the IAM policy that is attached to a service account. - - https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts/setIamPolicy - """ - resource_name = self.service_account_resource_name(account) - body = {"policy": policy.as_dict()} - logger.debug( - "Updating Service Account %s policy:\n%s", - account, - self.resource_pretty_format(body), - ) - try: - request: _HttpRequest = self._service_accounts.setIamPolicy( - resource=resource_name, body=body - ) - response: Dict[str, Any] = self._execute(request) - return Policy.from_response(response) - except gcp.api.ResponseError as error: - if error.status == 409: - # https://cloud.google.com/iam/docs/policies#etag - logger.debug(error) - raise EtagConflict from error - raise - - @handle_etag_conflict - def add_service_account_iam_policy_binding( - self, account: str, role: str, member: str - ) -> None: - """Add an IAM policy binding to an IAM service account. - - See for details on updating policy bindings: - https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts/setIamPolicy - """ - policy: Policy = self.get_service_account_iam_policy(account) - binding: Optional[Policy.Binding] = policy.find_binding_for_role(role) - if binding and member in binding.members: - logger.debug( - "Member %s already has role %s for Service Account %s", - member, - role, - account, - ) - return - - if binding is None: - updated_binding = Policy.Binding(role, frozenset([member])) - else: - updated_members: FrozenSet[str] = binding.members.union({member}) - updated_binding: Policy.Binding = ( - dataclasses.replace( # pylint: disable=too-many-function-args - binding, members=updated_members - ) - ) - - updated_policy: Policy = _replace_binding( - policy, binding, updated_binding - ) - self.set_service_account_iam_policy(account, updated_policy) - logger.debug( - "Role %s granted to member %s for Service Account %s", - role, - member, - account, - ) - - @handle_etag_conflict - def remove_service_account_iam_policy_binding( - self, account: str, role: str, member: str - ) -> None: - """Remove an IAM policy binding from the IAM policy of a service - account. - - See for details on updating policy bindings: - https://cloud.google.com/iam/docs/reference/rest/v1/projects.serviceAccounts/setIamPolicy - """ - policy: Policy = self.get_service_account_iam_policy(account) - binding: Optional[Policy.Binding] = policy.find_binding_for_role(role) - - if binding is None: - logger.debug( - "Noop: Service Account %s has no bindings for role %s", - account, - role, - ) - return - if member not in binding.members: - logger.debug( - "Noop: Service Account %s binding for role %s has no member %s", - account, - role, - member, - ) - return - - updated_members: FrozenSet[str] = binding.members.difference({member}) - updated_binding: Policy.Binding = ( - dataclasses.replace( # pylint: disable=too-many-function-args - binding, members=updated_members - ) - ) - updated_policy: Policy = _replace_binding( - policy, binding, updated_binding - ) - self.set_service_account_iam_policy(account, updated_policy) - logger.debug( - "Role %s revoked from member %s for Service Account %s", - role, - member, - account, - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_security.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_security.py deleted file mode 100644 index b7656bd45c9..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_security.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc -import dataclasses -import logging -from typing import Any, Dict - -from google.rpc import code_pb2 -import tenacity - -from framework.infrastructure import gcp - -logger = logging.getLogger(__name__) - -# Type aliases -GcpResource = gcp.compute.ComputeV1.GcpResource - - -@dataclasses.dataclass(frozen=True) -class ServerTlsPolicy: - url: str - name: str - server_certificate: dict - mtls_policy: dict - update_time: str - create_time: str - - @classmethod - def from_response( - cls, name: str, response: Dict[str, Any] - ) -> "ServerTlsPolicy": - return cls( - name=name, - url=response["name"], - server_certificate=response.get("serverCertificate", {}), - mtls_policy=response.get("mtlsPolicy", {}), - create_time=response["createTime"], - update_time=response["updateTime"], - ) - - -@dataclasses.dataclass(frozen=True) -class ClientTlsPolicy: - url: str - name: str - client_certificate: dict - server_validation_ca: list - update_time: str - create_time: str - - @classmethod - def from_response( - cls, name: str, response: Dict[str, Any] - ) -> "ClientTlsPolicy": - return cls( - name=name, - url=response["name"], - client_certificate=response.get("clientCertificate", {}), - server_validation_ca=response.get("serverValidationCa", []), - create_time=response["createTime"], - update_time=response["updateTime"], - ) - - -@dataclasses.dataclass(frozen=True) -class AuthorizationPolicy: - url: str - name: str - update_time: str - create_time: str - action: str - rules: list - - @classmethod - def from_response( - cls, name: str, response: Dict[str, Any] - ) -> "AuthorizationPolicy": - return cls( - name=name, - url=response["name"], - create_time=response["createTime"], - update_time=response["updateTime"], - action=response["action"], - rules=response.get("rules", []), - ) - - -class _NetworkSecurityBase( - gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta -): - """Base class for NetworkSecurity APIs.""" - - # TODO(https://github.com/grpc/grpc/issues/29532) remove pylint disable - # pylint: disable=abstract-method - - def __init__(self, api_manager: gcp.api.GcpApiManager, project: str): - super().__init__(api_manager.networksecurity(self.api_version), project) - # Shortcut to projects/*/locations/ endpoints - self._api_locations = self.api.projects().locations() - - @property - def api_name(self) -> str: - return "networksecurity" - - def _execute( - self, *args, **kwargs - ): # pylint: disable=signature-differs,arguments-differ - # Workaround TD bug: throttled operations are reported as internal. - # Ref b/175345578 - retryer = tenacity.Retrying( - retry=tenacity.retry_if_exception(self._operation_internal_error), - wait=tenacity.wait_fixed(10), - stop=tenacity.stop_after_delay(5 * 60), - before_sleep=tenacity.before_sleep_log(logger, logging.DEBUG), - reraise=True, - ) - retryer(super()._execute, *args, **kwargs) - - @staticmethod - def _operation_internal_error(exception): - return ( - isinstance(exception, gcp.api.OperationError) - and exception.error.code == code_pb2.INTERNAL - ) - - -class NetworkSecurityV1Beta1(_NetworkSecurityBase): - """NetworkSecurity API v1beta1.""" - - SERVER_TLS_POLICIES = "serverTlsPolicies" - CLIENT_TLS_POLICIES = "clientTlsPolicies" - AUTHZ_POLICIES = "authorizationPolicies" - - @property - def api_version(self) -> str: - return "v1beta1" - - def create_server_tls_policy(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.serverTlsPolicies(), - body=body, - serverTlsPolicyId=name, - ) - - def get_server_tls_policy(self, name: str) -> ServerTlsPolicy: - response = self._get_resource( - collection=self._api_locations.serverTlsPolicies(), - full_name=self.resource_full_name(name, self.SERVER_TLS_POLICIES), - ) - return ServerTlsPolicy.from_response(name, response) - - def delete_server_tls_policy(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.serverTlsPolicies(), - full_name=self.resource_full_name(name, self.SERVER_TLS_POLICIES), - ) - - def create_client_tls_policy(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.clientTlsPolicies(), - body=body, - clientTlsPolicyId=name, - ) - - def get_client_tls_policy(self, name: str) -> ClientTlsPolicy: - response = self._get_resource( - collection=self._api_locations.clientTlsPolicies(), - full_name=self.resource_full_name(name, self.CLIENT_TLS_POLICIES), - ) - return ClientTlsPolicy.from_response(name, response) - - def delete_client_tls_policy(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.clientTlsPolicies(), - full_name=self.resource_full_name(name, self.CLIENT_TLS_POLICIES), - ) - - def create_authz_policy(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.authorizationPolicies(), - body=body, - authorizationPolicyId=name, - ) - - def get_authz_policy(self, name: str) -> ClientTlsPolicy: - response = self._get_resource( - collection=self._api_locations.authorizationPolicies(), - full_name=self.resource_full_name(name, self.AUTHZ_POLICIES), - ) - return ClientTlsPolicy.from_response(name, response) - - def delete_authz_policy(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.authorizationPolicies(), - full_name=self.resource_full_name(name, self.AUTHZ_POLICIES), - ) - - -class NetworkSecurityV1Alpha1(NetworkSecurityV1Beta1): - """NetworkSecurity API v1alpha1. - - Note: extending v1beta1 class presumes that v1beta1 is just a v1alpha1 API - graduated into a more stable version. This is true in most cases. However, - v1alpha1 class can always override and reimplement incompatible methods. - """ - - @property - def api_version(self) -> str: - return "v1alpha1" diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py deleted file mode 100644 index 3ed21c534e3..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/gcp/network_services.py +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc -import dataclasses -import logging -from typing import Any, Dict, List, Optional, Tuple - -from google.rpc import code_pb2 -import tenacity - -from framework.infrastructure import gcp - -logger = logging.getLogger(__name__) - -# Type aliases -GcpResource = gcp.compute.ComputeV1.GcpResource - - -@dataclasses.dataclass(frozen=True) -class EndpointPolicy: - url: str - name: str - type: str - traffic_port_selector: dict - endpoint_matcher: dict - update_time: str - create_time: str - http_filters: Optional[dict] = None - server_tls_policy: Optional[str] = None - - @classmethod - def from_response( - cls, name: str, response: Dict[str, Any] - ) -> "EndpointPolicy": - return cls( - name=name, - url=response["name"], - type=response["type"], - server_tls_policy=response.get("serverTlsPolicy", None), - traffic_port_selector=response["trafficPortSelector"], - endpoint_matcher=response["endpointMatcher"], - http_filters=response.get("httpFilters", None), - update_time=response["updateTime"], - create_time=response["createTime"], - ) - - -@dataclasses.dataclass(frozen=True) -class Mesh: - name: str - url: str - routes: Optional[List[str]] - - @classmethod - def from_response(cls, name: str, d: Dict[str, Any]) -> "Mesh": - return cls( - name=name, - url=d["name"], - routes=list(d["routes"]) if "routes" in d else None, - ) - - -@dataclasses.dataclass(frozen=True) -class GrpcRoute: - @dataclasses.dataclass(frozen=True) - class MethodMatch: - type: Optional[str] - grpc_service: Optional[str] - grpc_method: Optional[str] - case_sensitive: Optional[bool] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.MethodMatch": - return cls( - type=d.get("type"), - grpc_service=d.get("grpcService"), - grpc_method=d.get("grpcMethod"), - case_sensitive=d.get("caseSensitive"), - ) - - @dataclasses.dataclass(frozen=True) - class HeaderMatch: - type: Optional[str] - key: str - value: str - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.HeaderMatch": - return cls( - type=d.get("type"), - key=d["key"], - value=d["value"], - ) - - @dataclasses.dataclass(frozen=True) - class RouteMatch: - method: Optional["GrpcRoute.MethodMatch"] - headers: Tuple["GrpcRoute.HeaderMatch"] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteMatch": - return cls( - method=GrpcRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - GrpcRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), - ) - - @dataclasses.dataclass(frozen=True) - class Destination: - service_name: str - weight: Optional[int] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.Destination": - return cls( - service_name=d["serviceName"], - weight=d.get("weight"), - ) - - @dataclasses.dataclass(frozen=True) - class RouteAction: - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteAction": - destinations = ( - [ - GrpcRoute.Destination.from_response(dest) - for dest in d["destinations"] - ] - if "destinations" in d - else [] - ) - return cls(destinations=destinations) - - @dataclasses.dataclass(frozen=True) - class RouteRule: - matches: List["GrpcRoute.RouteMatch"] - action: "GrpcRoute.RouteAction" - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteRule": - matches = ( - [GrpcRoute.RouteMatch.from_response(m) for m in d["matches"]] - if "matches" in d - else [] - ) - return cls( - matches=matches, - action=GrpcRoute.RouteAction.from_response(d["action"]), - ) - - name: str - url: str - hostnames: Tuple[str] - rules: Tuple["GrpcRoute.RouteRule"] - meshes: Optional[Tuple[str]] - - @classmethod - def from_response( - cls, name: str, d: Dict[str, Any] - ) -> "GrpcRoute.RouteRule": - return cls( - name=name, - url=d["name"], - hostnames=tuple(d["hostnames"]), - rules=tuple(d["rules"]), - meshes=None if d.get("meshes") is None else tuple(d["meshes"]), - ) - - -@dataclasses.dataclass(frozen=True) -class HttpRoute: - @dataclasses.dataclass(frozen=True) - class MethodMatch: - type: Optional[str] - http_service: Optional[str] - http_method: Optional[str] - case_sensitive: Optional[bool] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.MethodMatch": - return cls( - type=d.get("type"), - http_service=d.get("httpService"), - http_method=d.get("httpMethod"), - case_sensitive=d.get("caseSensitive"), - ) - - @dataclasses.dataclass(frozen=True) - class HeaderMatch: - type: Optional[str] - key: str - value: str - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.HeaderMatch": - return cls( - type=d.get("type"), - key=d["key"], - value=d["value"], - ) - - @dataclasses.dataclass(frozen=True) - class RouteMatch: - method: Optional["HttpRoute.MethodMatch"] - headers: Tuple["HttpRoute.HeaderMatch"] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteMatch": - return cls( - method=HttpRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None, - headers=tuple( - HttpRoute.HeaderMatch.from_response(h) for h in d["headers"] - ) - if "headers" in d - else (), - ) - - @dataclasses.dataclass(frozen=True) - class Destination: - service_name: str - weight: Optional[int] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.Destination": - return cls( - service_name=d["serviceName"], - weight=d.get("weight"), - ) - - @dataclasses.dataclass(frozen=True) - class RouteAction: - destinations: List["HttpRoute.Destination"] - stateful_session_affinity: Optional["HttpRoute.StatefulSessionAffinity"] - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteAction": - destinations = ( - [ - HttpRoute.Destination.from_response(dest) - for dest in d["destinations"] - ] - if "destinations" in d - else [] - ) - stateful_session_affinity = ( - HttpRoute.StatefulSessionAffinity.from_response( - d["statefulSessionAffinity"] - ) - if "statefulSessionAffinity" in d - else None - ) - return cls( - destinations=destinations, - stateful_session_affinity=stateful_session_affinity, - ) - - @dataclasses.dataclass(frozen=True) - class StatefulSessionAffinity: - cookie_ttl: Optional[str] - - @classmethod - def from_response( - cls, d: Dict[str, Any] - ) -> "HttpRoute.StatefulSessionAffinity": - return cls(cookie_ttl=d.get("cookieTtl")) - - @dataclasses.dataclass(frozen=True) - class RouteRule: - matches: List["HttpRoute.RouteMatch"] - action: "HttpRoute.RouteAction" - - @classmethod - def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteRule": - matches = ( - [HttpRoute.RouteMatch.from_response(m) for m in d["matches"]] - if "matches" in d - else [] - ) - return cls( - matches=matches, - action=HttpRoute.RouteAction.from_response(d["action"]), - ) - - name: str - url: str - hostnames: Tuple[str] - rules: Tuple["HttpRoute.RouteRule"] - meshes: Optional[Tuple[str]] - - @classmethod - def from_response(cls, name: str, d: Dict[str, Any]) -> "HttpRoute": - return cls( - name=name, - url=d["name"], - hostnames=tuple(d["hostnames"]), - rules=tuple(d["rules"]), - meshes=None if d.get("meshes") is None else tuple(d["meshes"]), - ) - - -class _NetworkServicesBase( - gcp.api.GcpStandardCloudApiResource, metaclass=abc.ABCMeta -): - """Base class for NetworkServices APIs.""" - - # TODO(https://github.com/grpc/grpc/issues/29532) remove pylint disable - # pylint: disable=abstract-method - - def __init__(self, api_manager: gcp.api.GcpApiManager, project: str): - super().__init__(api_manager.networkservices(self.api_version), project) - # Shortcut to projects/*/locations/ endpoints - self._api_locations = self.api.projects().locations() - - @property - def api_name(self) -> str: - return "networkservices" - - def _execute( - self, *args, **kwargs - ): # pylint: disable=signature-differs,arguments-differ - # Workaround TD bug: throttled operations are reported as internal. - # Ref b/175345578 - retryer = tenacity.Retrying( - retry=tenacity.retry_if_exception(self._operation_internal_error), - wait=tenacity.wait_fixed(10), - stop=tenacity.stop_after_delay(5 * 60), - before_sleep=tenacity.before_sleep_log(logger, logging.DEBUG), - reraise=True, - ) - retryer(super()._execute, *args, **kwargs) - - @staticmethod - def _operation_internal_error(exception): - return ( - isinstance(exception, gcp.api.OperationError) - and exception.error.code == code_pb2.INTERNAL - ) - - -class NetworkServicesV1Beta1(_NetworkServicesBase): - """NetworkServices API v1beta1.""" - - ENDPOINT_POLICIES = "endpointPolicies" - - @property - def api_version(self) -> str: - return "v1beta1" - - def create_endpoint_policy(self, name, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.endpointPolicies(), - body=body, - endpointPolicyId=name, - ) - - def get_endpoint_policy(self, name: str) -> EndpointPolicy: - response = self._get_resource( - collection=self._api_locations.endpointPolicies(), - full_name=self.resource_full_name(name, self.ENDPOINT_POLICIES), - ) - return EndpointPolicy.from_response(name, response) - - def delete_endpoint_policy(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.endpointPolicies(), - full_name=self.resource_full_name(name, self.ENDPOINT_POLICIES), - ) - - -class NetworkServicesV1Alpha1(NetworkServicesV1Beta1): - """NetworkServices API v1alpha1. - - Note: extending v1beta1 class presumes that v1beta1 is just a v1alpha1 API - graduated into a more stable version. This is true in most cases. However, - v1alpha1 class can always override and reimplement incompatible methods. - """ - - HTTP_ROUTES = "httpRoutes" - GRPC_ROUTES = "grpcRoutes" - MESHES = "meshes" - - @property - def api_version(self) -> str: - return "v1alpha1" - - def create_mesh(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.meshes(), body=body, meshId=name - ) - - def get_mesh(self, name: str) -> Mesh: - full_name = self.resource_full_name(name, self.MESHES) - result = self._get_resource( - collection=self._api_locations.meshes(), full_name=full_name - ) - return Mesh.from_response(name, result) - - def delete_mesh(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.meshes(), - full_name=self.resource_full_name(name, self.MESHES), - ) - - def create_grpc_route(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.grpcRoutes(), - body=body, - grpcRouteId=name, - ) - - def create_http_route(self, name: str, body: dict) -> GcpResource: - return self._create_resource( - collection=self._api_locations.httpRoutes(), - body=body, - httpRouteId=name, - ) - - def get_grpc_route(self, name: str) -> GrpcRoute: - full_name = self.resource_full_name(name, self.GRPC_ROUTES) - result = self._get_resource( - collection=self._api_locations.grpcRoutes(), full_name=full_name - ) - return GrpcRoute.from_response(name, result) - - def get_http_route(self, name: str) -> GrpcRoute: - full_name = self.resource_full_name(name, self.HTTP_ROUTES) - result = self._get_resource( - collection=self._api_locations.httpRoutes(), full_name=full_name - ) - return HttpRoute.from_response(name, result) - - def delete_grpc_route(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.grpcRoutes(), - full_name=self.resource_full_name(name, self.GRPC_ROUTES), - ) - - def delete_http_route(self, name: str) -> bool: - return self._delete_resource( - collection=self._api_locations.httpRoutes(), - full_name=self.resource_full_name(name, self.HTTP_ROUTES), - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s.py deleted file mode 100644 index 7643a3ef5b5..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s.py +++ /dev/null @@ -1,1152 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# TODO(sergiitk): to k8s/ package, and get rid of k8s_internal, which is only -# added to get around circular dependencies caused by k8s.py clashing with -# k8s/__init__.py -import datetime -import functools -import json -import logging -import pathlib -import threading -from typing import Any, Callable, List, Optional, Tuple -import warnings - -from kubernetes import client -from kubernetes import dynamic -from kubernetes import utils -import kubernetes.config -from kubernetes.dynamic import exceptions as dynamic_exc -from kubernetes.dynamic import resource as dynamic_res -import urllib3.exceptions -import yaml - -import framework.errors -from framework.helpers import retryers -import framework.helpers.datetime -import framework.helpers.highlighter -from framework.infrastructure.k8s_internal import k8s_log_collector -from framework.infrastructure.k8s_internal import k8s_port_forwarder - -logger = logging.getLogger(__name__) - -# Type aliases -PodLogCollector = k8s_log_collector.PodLogCollector -PortForwarder = k8s_port_forwarder.PortForwarder -V1Deployment = client.V1Deployment -V1ServiceAccount = client.V1ServiceAccount -V1Pod = client.V1Pod -V1PodList = client.V1PodList -V1Service = client.V1Service -V1Namespace = client.V1Namespace -V1ObjectMeta = client.V1ObjectMeta - -DynResourceInstance = dynamic_res.ResourceInstance -GammaMesh = DynResourceInstance -GammaGrpcRoute = DynResourceInstance -GammaHttpRoute = DynResourceInstance -GcpSessionAffinityPolicy = DynResourceInstance -GcpSessionAffinityFilter = DynResourceInstance -GcpBackendPolicy = DynResourceInstance - -_timedelta = datetime.timedelta -_datetime = datetime.datetime -_helper_datetime = framework.helpers.datetime -_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml -_ApiException = client.ApiException -_FailToCreateError = utils.FailToCreateError - -_RETRY_ON_EXCEPTIONS = ( - urllib3.exceptions.HTTPError, - _ApiException, - _FailToCreateError, -) - - -def _server_restart_retryer() -> retryers.Retrying: - return retryers.exponential_retryer_with_timeout( - retry_on_exceptions=_RETRY_ON_EXCEPTIONS, - wait_min=_timedelta(seconds=1), - wait_max=_timedelta(seconds=10), - timeout=_timedelta(minutes=3), - ) - - -def _too_many_requests_retryer() -> retryers.Retrying: - return retryers.exponential_retryer_with_timeout( - retry_on_exceptions=_RETRY_ON_EXCEPTIONS, - wait_min=_timedelta(seconds=10), - wait_max=_timedelta(seconds=30), - timeout=_timedelta(minutes=3), - ) - - -def _quick_recovery_retryer() -> retryers.Retrying: - return retryers.constant_retryer( - wait_fixed=_timedelta(seconds=1), - attempts=3, - retry_on_exceptions=_RETRY_ON_EXCEPTIONS, - ) - - -def label_dict_to_selector(labels: dict) -> str: - return ",".join(f"{k}=={v}" for k, v in labels.items()) - - -class NotFound(Exception): - """Indicates the resource is not found on the API server.""" - - -class KubernetesApiManager: - _client: client.ApiClient - _dynamic_client: dynamic.DynamicClient - context: str - apps: client.AppsV1Api - core: client.CoreV1Api - _apis: set[object] - _dynamic_apis: set[str] - - def __init__(self, context: str): - self.context = context - self._client = self._new_client_from_context(context) - self._dynamic_client = dynamic.DynamicClient(self._client) - self.apps = client.AppsV1Api(self.client) - self.core = client.CoreV1Api(self.client) - self._apis = {self.apps, self.core} - self._dynamic_apis = set() - # TODO(https://github.com/kubernetes-client/python/issues/2101): remove - # when the issue is solved, and the kubernetes dependency is bumped. - warnings.filterwarnings( - "ignore", - category=DeprecationWarning, - module="kubernetes", - message=( - "HTTPResponse.getheaders?\\(\\) is deprecated" - " and will be removed in urllib3 v2.1.0." - ), - ) - - @property - def client(self) -> client.ApiClient: - return self._client - - @property - def dynamic_client(self) -> dynamic.DynamicClient: - return self._dynamic_client - - @functools.cache # pylint: disable=no-member - def grpc_route(self, version: str) -> dynamic_res.Resource: - api_name = "gateway.networking.k8s.io" - kind = "GRPCRoute" - supported_versions = { - "v1alpha2", - } - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - @functools.cache # pylint: disable=no-member - def http_route(self, version: str) -> dynamic_res.Resource: - api_name = "gateway.networking.k8s.io" - kind = "HTTPRoute" - supported_versions = {"v1alpha2", "v1beta1"} - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - @functools.cache # pylint: disable=no-member - def gcp_session_affinity_filter(self, version: str) -> dynamic_res.Resource: - api_name = "networking.gke.io" - kind = "GCPSessionAffinityFilter" - supported_versions = ("v1",) - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - @functools.cache # pylint: disable=no-member - def gcp_session_affinity_policy(self, version: str) -> dynamic_res.Resource: - api_name = "networking.gke.io" - kind = "GCPSessionAffinityPolicy" - supported_versions = ("v1",) - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - @functools.cache # pylint: disable=no-member - def gcp_backend_policy(self, version: str) -> dynamic_res.Resource: - api_name = "networking.gke.io" - kind = "GCPBackendPolicy" - supported_versions = ("v1",) - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - @functools.cache # pylint: disable=no-member - def pod_monitoring(self, version: str) -> dynamic_res.Resource: - api_name = "monitoring.googleapis.com" - kind = "PodMonitoring" - supported_versions = ("v1",) - if version not in supported_versions: - raise NotImplementedError( - f"{kind} {api_name}/{version} not implemented." - ) - - return self._load_dynamic_api(api_name, version, kind) - - def close(self): - # TODO(sergiitk): [GAMMA] what to do with dynamic clients? - self.client.close() - - def reload(self): - self.close() - self._client = self._new_client_from_context(self.context) - # Update default configuration so that modules that initialize - # ApiClient implicitly (e.g. kubernetes.watch.Watch) get the updates. - client.Configuration.set_default(self._client.configuration) - for api in self._apis: - api.api_client = self._client - - # TODO(sergiitk): [GAMMA] what to do with dynamic apis? - - @classmethod - def _new_client_from_context(cls, context: str) -> "client.ApiClient": - client_instance = kubernetes.config.new_client_from_config( - context=context - ) - logger.info( - 'Using kubernetes context "%s", active host: %s', - context, - client_instance.configuration.host, - ) - # TODO(sergiitk): fine-tune if we see the total wait unreasonably long. - client_instance.configuration.retries = 10 - return client_instance - - def _load_dynamic_api( - self, api_name: str, version: str, kind: str - ) -> dynamic_res.Resource: - api_version = f"{api_name}/{version}" - try: - k8s_api: dynamic_res.Resource = self.dynamic_client.resources.get( - api_version=api_version, kind=kind - ) - self._dynamic_apis.add(k8s_api.group_version) - return k8s_api - except dynamic_exc.ResourceNotFoundError as err: - # TODO(sergiitk): [GAMMA] add retries if static client - # retries not apply. - raise RuntimeError( - f"Couldn't discover k8s API {api_version}, resource {kind}", - ) from err - - -class KubernetesNamespace: # pylint: disable=too-many-public-methods - _highlighter: framework.helpers.highlighter.Highlighter - _api: KubernetesApiManager - _name: str - - NEG_STATUS_ANNOTATION = "cloud.google.com/neg-status" - DELETE_GRACE_PERIOD_SEC: int = 5 - WAIT_SHORT_TIMEOUT_SEC: int = 60 - WAIT_SHORT_SLEEP_SEC: int = 1 - WAIT_MEDIUM_TIMEOUT_SEC: int = 5 * 60 - WAIT_MEDIUM_SLEEP_SEC: int = 10 - WAIT_LONG_TIMEOUT_SEC: int = 10 * 60 - WAIT_LONG_SLEEP_SEC: int = 30 - WAIT_POD_START_TIMEOUT_SEC: int = 3 * 60 - - # TODO(sergiitk): Find a better way. Maybe like in framework.rpc.grpc? - wait_for_namespace_deleted_timeout_sec = None - wait_for_namespace_deleted_sleep_sec = None - - def __init__(self, api: KubernetesApiManager, name: str): - self._api = api - self._name = name - self._highlighter = _HighlighterYaml() - - @property - def name(self): - return self._name - - @functools.cached_property # pylint: disable=no-member - def api_gke_mesh(self) -> dynamic_res.Resource: - return self._get_dynamic_api("net.gke.io/v1alpha1", "TDMesh") - - @functools.cached_property # pylint: disable=no-member - def api_grpc_route(self) -> dynamic_res.Resource: - return self._get_dynamic_api( - "gateway.networking.k8s.io/v1alpha2", - "GRPCRoute", - ) - - @functools.cached_property # pylint: disable=no-member - def api_http_route(self) -> dynamic_res.Resource: - return self._get_dynamic_api( - "gateway.networking.k8s.io/v1beta1", - "HTTPRoute", - ) - - @functools.cached_property # pylint: disable=no-member - def api_session_affinity_filter(self) -> dynamic_res.Resource: - return self._get_dynamic_api( - "networking.gke.io/v1", - "GCPSessionAffinityFilter", - ) - - @functools.cached_property # pylint: disable=no-member - def api_session_affinity_policy(self) -> dynamic_res.Resource: - return self._get_dynamic_api( - "networking.gke.io/v1", - "GCPSessionAffinityPolicy", - ) - - @functools.cached_property # pylint: disable=no-member - def api_backend_policy(self) -> dynamic_res.Resource: - return self._get_dynamic_api( - "networking.gke.io/v1", - "GCPBackendPolicy", - ) - - def _refresh_auth(self): - logger.info("Reloading k8s api client to refresh the auth.") - self._api.reload() - - def _apply_manifest_single(self, manifest) -> object: - k8s_objects = utils.create_from_dict( - self._api.client, manifest, namespace=self.name - ) - if len(k8s_objects) != 1: - raise ValueError( - f"Expected exactly one k8s object created from" - f" manifest {manifest}" - ) - return k8s_objects[0] - - def _apply_manifest_custom_object(self, manifest) -> DynResourceInstance: - api = self._get_dynamic_api(manifest["apiVersion"], manifest["kind"]) - return api.create(manifest) - - @functools.cache # pylint: disable=no-member - def _get_dynamic_api(self, api_version, kind) -> dynamic_res.Resource: - group, _, version = api_version.partition("/") - - # TODO(sergiitk): [GAMMA] Needs to be improved. This all is very clunky - # when considered together with _get_dynamic_api and api_gke_mesh, - # api_grpc_route. - if group == "networking.gke.io": - if kind == "GCPSessionAffinityFilter": - return self._api.gcp_session_affinity_filter(version) - elif kind == "GCPSessionAffinityPolicy": - return self._api.gcp_session_affinity_policy(version) - elif kind == "GCPBackendPolicy": - return self._api.gcp_backend_policy(version) - elif group == "gateway.networking.k8s.io": - if kind == "GRPCRoute": - return self._api.grpc_route(version) - elif kind == "HTTPRoute": - return self._api.http_route(version) - elif group == "monitoring.googleapis.com": - if kind == "PodMonitoring": - return self._api.pod_monitoring(version) - - raise NotImplementedError( - f"{kind} ({group}) {api_version} not implemented." - ) - - def _get_resource(self, method: Callable[[Any], object], *args, **kwargs): - try: - return self._execute(method, *args, **kwargs) - except NotFound: - # Instead of trowing an error when a resource doesn't exist, - # just return None. - return None - - def _get_dyn_resource( - self, api: dynamic_res.Resource, name, *args, **kwargs - ) -> Optional[DynResourceInstance]: - try: - return api.get(name=name, namespace=self.name, *args, **kwargs) - except dynamic_exc.NotFoundError: - # Instead of trowing an error when a resource doesn't exist, - # just return None. - return None - - def _execute(self, method: Callable[[Any], object], *args, **kwargs): - # Note: Intentionally leaving return type as unspecified to not confuse - # pytype for methods that delegate calls to this wrapper. - try: - return method(*args, **kwargs) - except _RETRY_ON_EXCEPTIONS as err: - retryer = self._handle_exception(err) - if retryer is not None: - return retryer(method, *args, **kwargs) - raise - - def _handle_exception(self, err: Exception) -> Optional[retryers.Retrying]: - # TODO(sergiitk): replace returns with match/case when we use to py3.10. - # pylint: disable=too-many-return-statements - - # Unwrap MaxRetryError. - if isinstance(err, urllib3.exceptions.MaxRetryError): - return self._handle_exception(err.reason) if err.reason else None - - # We consider all `NewConnectionError`s as caused by a k8s - # API server restart. `NewConnectionError`s we've seen: - # - [Errno 110] Connection timed out - # - [Errno 111] Connection refused - if isinstance(err, urllib3.exceptions.NewConnectionError): - return _server_restart_retryer() - - # We consider all `ProtocolError`s with "Connection aborted" message - # as caused by a k8s API server restart. - # `ProtocolError`s we've seen: - # - RemoteDisconnected('Remote end closed connection - # without response') - # - ConnectionResetError(104, 'Connection reset by peer') - if isinstance(err, urllib3.exceptions.ProtocolError): - if "connection aborted" in str(err).lower(): - return _server_restart_retryer() - else: - # To cover other cases we didn't account for, and haven't - # seen in the wild, f.e. "Connection broken" - return _quick_recovery_retryer() - - # ApiException means the server has received our request and responded - # with an error we can parse (except a few corner cases, f.e. SSLError). - if isinstance(err, _ApiException): - return self._handle_api_exception(err) - - # Unwrap FailToCreateError. - if isinstance(err, _FailToCreateError): - # We're always sending a single document, so we expect - # a single wrapped exception in return. - if len(err.api_exceptions) == 1: - return self._handle_exception(err.api_exceptions[0]) - - return None - - def _handle_api_exception( - self, err: _ApiException - ) -> Optional[retryers.Retrying]: - # TODO(sergiitk): replace returns with match/case when we use to py3.10. - # pylint: disable=too-many-return-statements - - # TODO(sergiitk): can I chain the retryers? - logger.debug( - "Handling k8s.ApiException: status=%s reason=%s body=%s headers=%s", - err.status, - err.reason, - err.body, - err.headers, - ) - - # TODO(sergiitk): [GAMMA] let dynamic/exception parse this instead? - code: int = err.status - body = err.body.lower() if err.body else "" - - # 401 Unauthorized: token might be expired, attempt auth refresh. - if code == 401: - self._refresh_auth() - return _quick_recovery_retryer() - - # 404 Not Found. Make it easier for the caller to handle 404s. - if code == 404: - raise NotFound( - "Kubernetes API returned 404 Not Found: " - f"{self._status_message_or_body(body)}" - ) from err - - # 409 Conflict - # "Operation cannot be fulfilled on resourcequotas "foo": the object - # has been modified; please apply your changes to the latest version - # and try again". - # See https://github.com/kubernetes/kubernetes/issues/67761 - if code == 409: - return _quick_recovery_retryer() - - # 429 Too Many Requests: "Too many requests, please try again later" - if code == 429: - return _too_many_requests_retryer() - - # 500 Internal Server Error - if code == 500: - # Observed when using `kubectl proxy`. - # "dial tcp 127.0.0.1:8080: connect: connection refused" - if "connection refused" in body: - return _server_restart_retryer() - - # Known 500 errors that should be treated as 429: - # - Internal Server Error: "/api/v1/namespaces": the server has - # received too many requests and has asked us - # to try again later - # - Internal Server Error: "/api/v1/namespaces/foo/services": - # the server is currently unable to handle the request - if ( - "too many requests" in body - or "currently unable to handle the request" in body - ): - return _too_many_requests_retryer() - - # In other cases, just retry a few times in case the server - # resumes normal operation. - return _quick_recovery_retryer() - - # 504 Gateway Timeout: - # "Timeout: request did not complete within the allotted timeout" - if code == 504: - return _server_restart_retryer() - - return None - - @classmethod - def _status_message_or_body(cls, body: str) -> str: - try: - return str(json.loads(body)["message"]) - except (KeyError, ValueError): - return body - - def create_single_resource(self, manifest, custom_object: bool = False): - if custom_object: - return self._execute(self._apply_manifest_custom_object, manifest) - - return self._execute(self._apply_manifest_single, manifest) - - def get_service(self, name) -> V1Service: - return self._get_resource( - self._api.core.read_namespaced_service, name, self.name - ) - - def get_gamma_mesh(self, name) -> Optional[GammaMesh]: - return self._get_dyn_resource(self.api_gke_mesh, name) - - def get_gamma_route(self, name) -> Optional[GammaHttpRoute]: - return self._get_dyn_resource(self.api_http_route, name) - - def get_session_affinity_policy( - self, name - ) -> Optional[GcpSessionAffinityPolicy]: - return self._get_dyn_resource(self.api_session_affinity_policy, name) - - def get_session_affinity_filter( - self, name - ) -> Optional[GcpSessionAffinityFilter]: - return self._get_dyn_resource(self.api_session_affinity_filter, name) - - def get_backend_policy(self, name) -> Optional[GcpBackendPolicy]: - return self._get_dyn_resource(self.api_backend_policy, name) - - def get_service_account(self, name) -> V1Service: - return self._get_resource( - self._api.core.read_namespaced_service_account, name, self.name - ) - - def delete_service( - self, name, grace_period_seconds=DELETE_GRACE_PERIOD_SEC - ): - self._execute( - self._api.core.delete_namespaced_service, - name=name, - namespace=self.name, - body=client.V1DeleteOptions( - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ), - ) - - def delete_service_account( - self, name, grace_period_seconds=DELETE_GRACE_PERIOD_SEC - ): - self._execute( - self._api.core.delete_namespaced_service_account, - name=name, - namespace=self.name, - body=client.V1DeleteOptions( - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ), - ) - - def delete_gamma_mesh( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - # TODO(sergiitk): [GAMMA] Can we call delete on dynamic_res.ResourceList - # to avoid no-member issues due to dynamic_res.Resource proxying calls? - self._execute( - self.api_gke_mesh.delete, # pylint: disable=no-member - name=name, - namespace=self.name, - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ) - - def delete_gamma_route( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - # TODO(sergiitk): [GAMMA] Can we call delete on dynamic_res.ResourceList - # to avoid no-member issues due to dynamic_res.Resource proxying calls? - self._execute( - self.api_http_route.delete, # pylint: disable=no-member - name=name, - namespace=self.name, - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ) - - def delete_session_affinity_policy( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - self._execute( - self.api_session_affinity_policy.delete, # pylint: disable=no-member - name=name, - namespace=self.name, - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ) - - def delete_session_affinity_filter( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - self._execute( - self.api_session_affinity_filter.delete, # pylint: disable=no-member - name=name, - namespace=self.name, - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ) - - def delete_backend_policy( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - self._execute( - self.api_backend_policy.delete, # pylint: disable=no-member - name=name, - namespace=self.name, - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ) - - def delete_pod_async( - self, - name: str, - grace_period_seconds=DELETE_GRACE_PERIOD_SEC, - ) -> None: - # TODO(sergiitk): Do we need async? Won't it break error handling? - self._execute( - self._api.core.delete_namespaced_pod, - name=name, - namespace=self.name, - body=client.V1DeleteOptions( - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ), - async_req=True, - ) - - def get(self) -> V1Namespace: - return self._get_resource(self._api.core.read_namespace, self.name) - - def delete(self, grace_period_seconds=DELETE_GRACE_PERIOD_SEC): - self._execute( - self._api.core.delete_namespace, - name=self.name, - body=client.V1DeleteOptions( - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ), - ) - - def wait_for_service_deleted( - self, - name: str, - timeout_sec: int = WAIT_SHORT_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda service: service is None, - ) - retryer(self.get_service, name) - - def wait_for_get_gamma_mesh_deleted( - self, - name: str, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_MEDIUM_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda mesh: mesh is None, - ) - retryer(self.get_gamma_mesh, name) - - def wait_for_get_gamma_route_deleted( - self, - name: str, - timeout_sec: int = WAIT_SHORT_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda route: route is None, - ) - retryer(self.get_gamma_route, name) - - def wait_for_get_session_affinity_policy_deleted( - self, - name: str, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_MEDIUM_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda affinity_policy: affinity_policy is None, - ) - retryer(self.get_session_affinity_policy, name) - - def wait_for_get_session_affinity_filter_deleted( - self, - name: str, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_MEDIUM_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda affinity_filter: affinity_filter is None, - ) - retryer(self.get_session_affinity_filter, name) - - def wait_for_get_backend_policy_deleted( - self, - name: str, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_MEDIUM_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda backend_policy: backend_policy is None, - ) - retryer(self.get_backend_policy, name) - - def wait_for_service_account_deleted( - self, - name: str, - timeout_sec: int = WAIT_SHORT_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda service_account: service_account is None, - ) - retryer(self.get_service_account, name) - - def wait_for_namespace_deleted( - self, - timeout_sec: Optional[int] = None, - wait_sec: Optional[int] = None, - ) -> None: - if timeout_sec is None: - if self.wait_for_namespace_deleted_timeout_sec is not None: - timeout_sec = self.wait_for_namespace_deleted_timeout_sec - else: - timeout_sec = self.WAIT_LONG_TIMEOUT_SEC - if wait_sec is None: - if self.wait_for_namespace_deleted_sleep_sec is not None: - wait_sec = self.wait_for_namespace_deleted_timeout_sec - else: - wait_sec = self.WAIT_LONG_SLEEP_SEC - - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda namespace: namespace is None, - ) - retryer(self.get) - - def wait_for_service_neg_status_annotation( - self, - name: str, - timeout_sec: int = WAIT_SHORT_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - timeout = _timedelta(seconds=timeout_sec) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=timeout, - check_result=self._check_service_neg_status_annotation, - ) - try: - retryer(self.get_service, name) - except retryers.RetryError as retry_err: - result = retry_err.result() - note = framework.errors.FrameworkError.note_blanket_error_info_below( - "A Kubernetes Service wasn't assigned a NEG (Network Endpoint" - " Group) status annotation.", - info_below=( - f"Timeout {timeout} (h:mm:ss) waiting for Kubernetes" - f" Service {name} in the namespace {self.name} to report" - f" the '{self.NEG_STATUS_ANNOTATION}' metadata annotation." - f"\nThis indicates the NEG wasn't created OR" - f" the NEG creation event hasn't propagated to Kubernetes." - f" Service metadata:\n" - f"{self.pretty_format_metadata(result, highlight=False)}" - f"Service status:\n" - f"{self.pretty_format_status(result, highlight=False)}" - ), - ) - retry_err.add_note(note) - raise - - def parse_service_neg_status( - self, service_name: str, service_port: int - ) -> Tuple[str, List[str]]: - service = self.get_service(service_name) - neg_info: dict = json.loads( - service.metadata.annotations[self.NEG_STATUS_ANNOTATION] - ) - neg_name: str = neg_info["network_endpoint_groups"][str(service_port)] - neg_zones: List[str] = neg_info["zones"] - return neg_name, neg_zones - - def get_deployment(self, name) -> V1Deployment: - return self._get_resource( - self._api.apps.read_namespaced_deployment, name, self.name - ) - - def delete_deployment( - self, name: str, grace_period_seconds: int = DELETE_GRACE_PERIOD_SEC - ) -> None: - self._execute( - self._api.apps.delete_namespaced_deployment, - name=name, - namespace=self.name, - body=client.V1DeleteOptions( - propagation_policy="Foreground", - grace_period_seconds=grace_period_seconds, - ), - ) - - def list_deployment_pods(self, deployment: V1Deployment) -> List[V1Pod]: - # V1LabelSelector.match_expressions not supported at the moment - return self.list_pods_with_labels(deployment.spec.selector.match_labels) - - def wait_for_deployment_available_replicas( - self, - name: str, - count: int = 1, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - timeout = _timedelta(seconds=timeout_sec) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=timeout, - check_result=lambda depl: self._replicas_available(depl, count), - ) - try: - retryer(self.get_deployment, name) - except retryers.RetryError as retry_err: - result = retry_err.result() - note = framework.errors.FrameworkError.note_blanket_error_info_below( - "The deployment didn't report one or several pods available" - " (ready for at least minReadySeconds).", - info_below=( - f"Timeout {timeout} (h:mm:ss) waiting for deployment {name}" - f" to report {count} replicas available. Last status:\n" - f"{self.pretty_format_status(result, highlight=False)}" - ), - ) - retry_err.add_note(note) - raise - - def wait_for_deployment_replica_count( - self, - deployment: V1Deployment, - count: int = 1, - *, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - timeout = _timedelta(seconds=timeout_sec) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=timeout, - check_result=lambda pods: len(pods) == count, - ) - try: - retryer(self.list_deployment_pods, deployment) - except retryers.RetryError as retry_err: - result = retry_err.result(default=[]) - note = framework.errors.FrameworkError.note_blanket_error_info_below( - "The deployment was unable to initialize one or several pods.", - info_below=( - f"Timeout {timeout} (h:mm:ss) waiting for pod count" - f" {count}, got: {len(result)}. Pod statuses:\n" - f"{self.pretty_format_status(result, highlight=False)}" - ), - ) - retry_err.add_note(note) - raise - - def wait_for_deployment_deleted( - self, - deployment_name: str, - timeout_sec: int = WAIT_MEDIUM_TIMEOUT_SEC, - wait_sec: int = WAIT_MEDIUM_SLEEP_SEC, - ) -> None: - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=_timedelta(seconds=timeout_sec), - check_result=lambda deployment: deployment is None, - ) - retryer(self.get_deployment, deployment_name) - - def list_pods_with_labels(self, labels: dict) -> List[V1Pod]: - pod_list: V1PodList = self._execute( - self._api.core.list_namespaced_pod, - self.name, - label_selector=label_dict_to_selector(labels), - ) - return pod_list.items - - def get_pod(self, name: str) -> V1Pod: - return self._get_resource( - self._api.core.read_namespaced_pod, name, self.name - ) - - def wait_for_pod_started( - self, - pod_name: str, - timeout_sec: int = WAIT_POD_START_TIMEOUT_SEC, - wait_sec: int = WAIT_SHORT_SLEEP_SEC, - ) -> None: - timeout = _timedelta(seconds=timeout_sec) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=wait_sec), - timeout=timeout, - check_result=self._pod_started, - ) - try: - retryer(self.get_pod, pod_name) - except retryers.RetryError as retry_err: - result = retry_err.result() - retry_err.add_note( - framework.errors.FrameworkError.note_blanket_error_info_below( - "The pod didn't start within expected timeout.", - info_below=( - f"Timeout {timeout} (h:mm:ss) waiting for pod" - f" {pod_name} to start. Pod status:\n" - f"{self.pretty_format_status(result, highlight=False)}" - ), - ) - ) - raise - - def port_forward_pod( - self, - pod: V1Pod, - remote_port: int, - local_port: Optional[int] = None, - local_address: Optional[str] = None, - ) -> k8s_port_forwarder.PortForwarder: - pf = k8s_port_forwarder.PortForwarder( - self._api.context, - self.name, - f"pod/{pod.metadata.name}", - remote_port, - local_port, - local_address, - ) - pf.connect() - return pf - - def pod_start_logging( - self, - *, - pod_name: str, - log_path: pathlib.Path, - log_stop_event: threading.Event, - log_to_stdout: bool = False, - log_timestamps: bool = False, - ) -> PodLogCollector: - pod_log_collector = PodLogCollector( - pod_name=pod_name, - namespace_name=self.name, - read_pod_log_fn=self._api.core.read_namespaced_pod_log, - stop_event=log_stop_event, - log_path=log_path, - log_to_stdout=log_to_stdout, - log_timestamps=log_timestamps, - ) - pod_log_collector.start() - return pod_log_collector - - def pretty_format_statuses( - self, - k8s_objects: List[Optional[object]], - *, - highlight: bool = True, - ) -> str: - return "\n".join( - self.pretty_format_status(k8s_object, highlight=highlight) - for k8s_object in k8s_objects - ) - - def pretty_format_status( - self, - k8s_object: Optional[object], - highlight: bool = True, - ) -> str: - if k8s_object is None: - return "No data" - - result = [] - metadata: Optional[V1ObjectMeta] = None - if isinstance(getattr(k8s_object, "metadata", None), V1ObjectMeta): - # Parse the name if present. - metadata: V1ObjectMeta = k8s_object.metadata - - # Parse name if, present, but always indicate unsuccessful parse. - name = metadata.name if metadata else "Can't parse resource name" - result.append(f"Resource name: {name}") - - # Add kubernetes kind (resource type) if present. - if hasattr(k8s_object, "kind"): - result.append(f"Resource kind: {k8s_object.kind}") - - # Add the timestamps if present. - if metadata and metadata.creation_timestamp: - result.append( - f"Created: {metadata.creation_timestamp};" - f" {_helper_datetime.ago(metadata.creation_timestamp)}" - ) - if metadata and metadata.deletion_timestamp: - result.append( - f"Deletion requested: {metadata.deletion_timestamp};" - f" {_helper_datetime.ago(metadata.deletion_timestamp)}" - ) - - # Pretty-print the status if present. - result.append("") - if hasattr(k8s_object, "status"): - try: - status = self._pretty_format( - k8s_object.status.to_dict(), - highlight=highlight, - ) - except Exception as e: # pylint: disable=broad-except - # Catching all exceptions because not printing the status - # isn't as important as the system under test. - status = f"Can't parse resource status: {e}" - else: - status = "Can't parse resource status" - result.append(status) - - return "\n".join(result) + "\n" - - def pretty_format_metadata( - self, - k8s_object: Optional[object], - *, - highlight: bool = True, - managed_fields: bool = False, - ) -> str: - if k8s_object is None: - return "No data" - - # Parse the name if present. - if not hasattr(k8s_object, "metadata"): - return "Object metadata missing" - - name = k8s_object.metadata.name or "Can't parse resource name" - - # Pretty-print metadata. - try: - metadata_dict: dict = k8s_object.metadata.to_dict() - # Don't print manged fields by default. Lots of noise with no value. - if not managed_fields: - metadata_dict.pop("managed_fields", None) - metadata = self._pretty_format(metadata_dict, highlight=highlight) - except Exception as e: # pylint: disable=broad-except - # Catching all exceptions because not printing the metadata - # isn't as important as the system under test. - metadata = f"Can't parse resource metadata: {e}" - - # Return the name of k8s object, and its pretty-printed status. - return f"{name}:\n{metadata}\n" - - def _pretty_format( - self, - data: dict, - *, - highlight: bool = True, - ) -> str: - """Return a string with pretty-printed yaml data from a python dict.""" - yaml_out: str = yaml.dump(data, explicit_start=True, explicit_end=True) - return self._highlighter.highlight(yaml_out) if highlight else yaml_out - - @classmethod - def _check_service_neg_status_annotation( - cls, service: Optional[V1Service] - ) -> bool: - return ( - isinstance(service, V1Service) - and cls.NEG_STATUS_ANNOTATION in service.metadata.annotations - ) - - @classmethod - def _pod_started(cls, pod: V1Pod) -> bool: - return isinstance(pod, V1Pod) and pod.status.phase not in ( - "Pending", - "Unknown", - ) - - @classmethod - def _replicas_available(cls, deployment: V1Deployment, count: int) -> bool: - return ( - isinstance(deployment, V1Deployment) - and deployment.status.available_replicas is not None - and deployment.status.available_replicas >= count - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/__init__.py deleted file mode 100644 index a87ddec1edd..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_log_collector.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_log_collector.py deleted file mode 100644 index 4f0c287633f..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_log_collector.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import os -import pathlib -import threading -from typing import Any, Callable, Optional, TextIO - -from kubernetes import client -from kubernetes.watch import watch - -logger = logging.getLogger(__name__) - - -class PodLogCollector(threading.Thread): - """A thread that streams logs from the remote pod to a local file.""" - - pod_name: str - namespace_name: str - stop_event: threading.Event - drain_event: threading.Event - log_path: pathlib.Path - log_to_stdout: bool - log_timestamps: bool - error_backoff_sec: int - _out_stream: Optional[TextIO] - _watcher: Optional[watch.Watch] - _read_pod_log_fn: Callable[..., Any] - - def __init__( - self, - *, - pod_name: str, - namespace_name: str, - read_pod_log_fn: Callable[..., Any], - stop_event: threading.Event, - log_path: pathlib.Path, - log_to_stdout: bool = False, - log_timestamps: bool = False, - error_backoff_sec: int = 5, - ): - self.pod_name = pod_name - self.namespace_name = namespace_name - self.stop_event = stop_event - # Used to indicate log draining happened. Turned out to be not as useful - # in cases when the logging happens rarely because the blocking happens - # in the native code, which doesn't yield until the next log message. - self.drain_event = threading.Event() - self.log_path = log_path - self.log_to_stdout = log_to_stdout - self.log_timestamps = log_timestamps - self.error_backoff_sec = error_backoff_sec - self._read_pod_log_fn = read_pod_log_fn - self._out_stream = None - self._watcher = None - super().__init__(name=f"pod-log-{pod_name}", daemon=True) - - def run(self): - logger.info( - "Starting log collection thread %i for %s", - self.ident, - self.pod_name, - ) - try: - self._out_stream = open( - self.log_path, "w", errors="ignore", encoding="utf-8" - ) - while not self.stop_event.is_set(): - self._stream_log() - finally: - self._stop() - - def flush(self): - """Flushes the log file buffer. May be called from the main thread.""" - if self._out_stream: - self._out_stream.flush() - os.fsync(self._out_stream.fileno()) - - def _stop(self): - if self._watcher is not None: - self._watcher.stop() - self._watcher = None - if self._out_stream is not None: - self._write( - f"Finished log collection for pod {self.pod_name}", - force_flush=True, - ) - self._out_stream.close() - self._out_stream = None - self.drain_event.set() - - def _stream_log(self): - try: - self._restart_stream() - except client.ApiException as e: - self._write(f"Exception fetching logs: {e}") - self._write( - ( - f"Restarting log fetching in {self.error_backoff_sec} sec. " - "Will attempt to read from the beginning, but log " - "truncation may occur." - ), - force_flush=True, - ) - finally: - # Instead of time.sleep(), we're waiting on the stop event - # in case it gets set earlier. - self.stop_event.wait(timeout=self.error_backoff_sec) - - def _restart_stream(self): - self._watcher = watch.Watch() - for msg in self._watcher.stream( - self._read_pod_log_fn, - name=self.pod_name, - namespace=self.namespace_name, - timestamps=self.log_timestamps, - follow=True, - ): - self._write(msg) - # Every message check if a stop is requested. - if self.stop_event.is_set(): - self._stop() - return - - def _write(self, msg: str, force_flush: bool = False): - self._out_stream.write(msg) - self._out_stream.write("\n") - if force_flush: - self.flush() - if self.log_to_stdout: - logger.info(msg) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py deleted file mode 100644 index d43c9a4f0af..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/k8s_internal/k8s_port_forwarder.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import re -import subprocess -import time -from typing import Optional - -logger = logging.getLogger(__name__) - - -class PortForwardingError(Exception): - """Error forwarding port""" - - -class PortForwarder: - PORT_FORWARD_LOCAL_ADDRESS: str = "127.0.0.1" - - def __init__( - self, - context: str, - namespace: str, - destination: str, - remote_port: int, - local_port: Optional[int] = None, - local_address: Optional[str] = None, - ): - self.context = context - self.namespace = namespace - self.destination = destination - self.remote_port = remote_port - self.local_address = local_address or self.PORT_FORWARD_LOCAL_ADDRESS - self.local_port: Optional[int] = local_port - self.subprocess: Optional[subprocess.Popen] = None - - def connect(self) -> None: - if self.local_port: - port_mapping = f"{self.local_port}:{self.remote_port}" - else: - port_mapping = f":{self.remote_port}" - cmd = [ - "kubectl", - "--context", - self.context, - "--namespace", - self.namespace, - "port-forward", - "--address", - self.local_address, - self.destination, - port_mapping, - ] - logger.debug( - "Executing port forwarding subprocess cmd: %s", " ".join(cmd) - ) - self.subprocess = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - # Wait for stdout line indicating successful start. - if self.local_port: - local_port_expected = ( - f"Forwarding from {self.local_address}:{self.local_port}" - f" -> {self.remote_port}" - ) - else: - local_port_re = re.compile( - f"Forwarding from {self.local_address}:([0-9]+) ->" - f" {self.remote_port}" - ) - try: - while True: - time.sleep(0.05) - output = self.subprocess.stdout.readline().strip() - if not output: - return_code = self.subprocess.poll() - if return_code is not None: - errors = [ - error - for error in self.subprocess.stdout.readlines() - ] - raise PortForwardingError( - "Error forwarding port, kubectl return " - f"code {return_code}, output {errors}" - ) - # If there is no output, and the subprocess is not exiting, - # continue waiting for the log line. - continue - - # Validate output log - if self.local_port: - if output != local_port_expected: - raise PortForwardingError( - f"Error forwarding port, unexpected output {output}" - ) - else: - groups = local_port_re.search(output) - if groups is None: - raise PortForwardingError( - f"Error forwarding port, unexpected output {output}" - ) - # Update local port to the randomly picked one - self.local_port = int(groups[1]) - - logger.info(output) - break - except Exception: - self.close() - raise - - def close(self) -> None: - if self.subprocess is not None: - logger.info( - "Shutting down port forwarding, pid %s", self.subprocess.pid - ) - self.subprocess.kill() - stdout, _ = self.subprocess.communicate(timeout=5) - logger.info("Port forwarding stopped") - logger.debug("Port forwarding remaining stdout: %s", stdout) - self.subprocess = None diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py deleted file mode 100644 index bc549bbcb30..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director.py +++ /dev/null @@ -1,1118 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import functools -import logging -import random -from typing import Any, Dict, List, Optional, Set - -from framework import xds_flags -from framework.infrastructure import gcp - -logger = logging.getLogger(__name__) - -# Type aliases -# Compute -_ComputeV1 = gcp.compute.ComputeV1 -GcpResource = _ComputeV1.GcpResource -HealthCheckProtocol = _ComputeV1.HealthCheckProtocol -ZonalGcpResource = _ComputeV1.ZonalGcpResource -BackendServiceProtocol = _ComputeV1.BackendServiceProtocol -_BackendGRPC = BackendServiceProtocol.GRPC -_HealthCheckGRPC = HealthCheckProtocol.GRPC - -# Network Security -_NetworkSecurityV1Beta1 = gcp.network_security.NetworkSecurityV1Beta1 -ServerTlsPolicy = gcp.network_security.ServerTlsPolicy -ClientTlsPolicy = gcp.network_security.ClientTlsPolicy -AuthorizationPolicy = gcp.network_security.AuthorizationPolicy - -# Network Services -_NetworkServicesV1Alpha1 = gcp.network_services.NetworkServicesV1Alpha1 -_NetworkServicesV1Beta1 = gcp.network_services.NetworkServicesV1Beta1 -EndpointPolicy = gcp.network_services.EndpointPolicy -GrpcRoute = gcp.network_services.GrpcRoute -HttpRoute = gcp.network_services.HttpRoute -Mesh = gcp.network_services.Mesh - -# Testing metadata consts -TEST_AFFINITY_METADATA_KEY = "xds_md" - - -class TrafficDirectorManager: # pylint: disable=too-many-public-methods - compute: _ComputeV1 - resource_prefix: str - resource_suffix: str - - BACKEND_SERVICE_NAME = "backend-service" - ALTERNATIVE_BACKEND_SERVICE_NAME = "backend-service-alt" - AFFINITY_BACKEND_SERVICE_NAME = "backend-service-affinity" - HEALTH_CHECK_NAME = "health-check" - URL_MAP_NAME = "url-map" - ALTERNATIVE_URL_MAP_NAME = "url-map-alt" - URL_MAP_PATH_MATCHER_NAME = "path-matcher" - TARGET_PROXY_NAME = "target-proxy" - ALTERNATIVE_TARGET_PROXY_NAME = "target-proxy-alt" - FORWARDING_RULE_NAME = "forwarding-rule" - ALTERNATIVE_FORWARDING_RULE_NAME = "forwarding-rule-alt" - FIREWALL_RULE_NAME = "allow-health-checks" - - def __init__( - self, - gcp_api_manager: gcp.api.GcpApiManager, - project: str, - *, - resource_prefix: str, - resource_suffix: str, - network: str = "default", - compute_api_version: str = "v1", - ): - # API - self.compute = _ComputeV1( - gcp_api_manager, - project, - version=compute_api_version, - gfe_debug_header=xds_flags.GFE_DEBUG_HEADER.value, - ) - - # Settings - self.project: str = project - self.network: str = network - self.resource_prefix: str = resource_prefix - self.resource_suffix: str = resource_suffix - - # Managed resources - self.health_check: Optional[GcpResource] = None - self.backend_service: Optional[GcpResource] = None - # TODO(sergiitk): remove this flag once backend service resource loaded - self.backend_service_protocol: Optional[BackendServiceProtocol] = None - self.url_map: Optional[GcpResource] = None - self.alternative_url_map: Optional[GcpResource] = None - self.firewall_rule: Optional[GcpResource] = None - self.target_proxy: Optional[GcpResource] = None - # TODO(sergiitk): remove this flag once target proxy resource loaded - self.target_proxy_is_http: bool = False - self.alternative_target_proxy: Optional[GcpResource] = None - self.forwarding_rule: Optional[GcpResource] = None - self.alternative_forwarding_rule: Optional[GcpResource] = None - self.backends: Set[ZonalGcpResource] = set() - self.alternative_backend_service: Optional[GcpResource] = None - # TODO(sergiitk): remove this flag once backend service resource loaded - self.alternative_backend_service_protocol: Optional[ - BackendServiceProtocol - ] = None - self.alternative_backends: Set[ZonalGcpResource] = set() - self.affinity_backend_service: Optional[GcpResource] = None - # TODO(sergiitk): remove this flag once backend service resource loaded - self.affinity_backend_service_protocol: Optional[ - BackendServiceProtocol - ] = None - self.affinity_backends: Set[ZonalGcpResource] = set() - - @property - def network_url(self): - return f"global/networks/{self.network}" - - def setup_for_grpc( - self, - service_host, - service_port, - *, - backend_protocol: Optional[BackendServiceProtocol] = _BackendGRPC, - health_check_port: Optional[int] = None, - ): - self.setup_backend_for_grpc( - protocol=backend_protocol, health_check_port=health_check_port - ) - self.setup_routing_rule_map_for_grpc(service_host, service_port) - - def setup_backend_for_grpc( - self, - *, - protocol: Optional[BackendServiceProtocol] = _BackendGRPC, - health_check_port: Optional[int] = None, - ): - self.create_health_check(port=health_check_port) - self.create_backend_service(protocol) - - def setup_routing_rule_map_for_grpc(self, service_host, service_port): - self.create_url_map(service_host, service_port) - self.create_target_proxy() - self.create_forwarding_rule(service_port) - - def cleanup(self, *, force=False): - # Cleanup in the reverse order of creation - self.delete_forwarding_rule(force=force) - self.delete_alternative_forwarding_rule(force=force) - self.delete_target_http_proxy(force=force) - self.delete_target_grpc_proxy(force=force) - self.delete_alternative_target_grpc_proxy(force=force) - self.delete_url_map(force=force) - self.delete_alternative_url_map(force=force) - self.delete_backend_service(force=force) - self.delete_alternative_backend_service(force=force) - self.delete_affinity_backend_service(force=force) - self.delete_health_check(force=force) - - @functools.lru_cache(None) - def make_resource_name(self, name: str) -> str: - """Make dash-separated resource name with resource prefix and suffix.""" - parts = [self.resource_prefix, name] - # Avoid trailing dash when the suffix is empty. - if self.resource_suffix: - parts.append(self.resource_suffix) - return "-".join(parts) - - def create_health_check( - self, - *, - protocol: Optional[HealthCheckProtocol] = _HealthCheckGRPC, - port: Optional[int] = None, - ): - if self.health_check: - raise ValueError( - f"Health check {self.health_check.name} " - "already created, delete it first" - ) - if protocol is None: - protocol = _HealthCheckGRPC - - name = self.make_resource_name(self.HEALTH_CHECK_NAME) - logger.info('Creating %s Health Check "%s"', protocol.name, name) - resource = self.compute.create_health_check(name, protocol, port=port) - self.health_check = resource - - def delete_health_check(self, force=False): - if force: - name = self.make_resource_name(self.HEALTH_CHECK_NAME) - elif self.health_check: - name = self.health_check.name - else: - return - logger.info('Deleting Health Check "%s"', name) - self.compute.delete_health_check(name) - self.health_check = None - - def create_backend_service( - self, - protocol: Optional[BackendServiceProtocol] = _BackendGRPC, - subset_size: Optional[int] = None, - affinity_header: Optional[str] = None, - locality_lb_policies: Optional[List[dict]] = None, - outlier_detection: Optional[dict] = None, - ): - if protocol is None: - protocol = _BackendGRPC - - name = self.make_resource_name(self.BACKEND_SERVICE_NAME) - logger.info('Creating %s Backend Service "%s"', protocol.name, name) - resource = self.compute.create_backend_service_traffic_director( - name, - health_check=self.health_check, - protocol=protocol, - subset_size=subset_size, - affinity_header=affinity_header, - locality_lb_policies=locality_lb_policies, - outlier_detection=outlier_detection, - ) - self.backend_service = resource - self.backend_service_protocol = protocol - - def load_backend_service(self): - name = self.make_resource_name(self.BACKEND_SERVICE_NAME) - resource = self.compute.get_backend_service_traffic_director(name) - self.backend_service = resource - - def delete_backend_service(self, force=False): - if force: - name = self.make_resource_name(self.BACKEND_SERVICE_NAME) - elif self.backend_service: - name = self.backend_service.name - else: - return - logger.info('Deleting Backend Service "%s"', name) - self.compute.delete_backend_service(name) - self.backend_service = None - - def backend_service_add_neg_backends( - self, name, zones, max_rate_per_endpoint: Optional[int] = None - ): - self.backend_service_load_neg_backends(name, zones) - if not self.backends: - raise ValueError("Unexpected: no backends were loaded.") - self.backend_service_patch_backends(max_rate_per_endpoint) - - def backend_service_load_neg_backends(self, name, zones): - logger.info("Waiting for Network Endpoint Groups to load endpoints.") - for zone in zones: - backend = self.compute.wait_for_network_endpoint_group(name, zone) - logger.info( - 'Loaded NEG "%s" in zone %s', backend.name, backend.zone - ) - self.backends.add(backend) - - def backend_service_remove_neg_backends(self, name, zones): - logger.info("Waiting for Network Endpoint Groups to load endpoints.") - for zone in zones: - backend = self.compute.wait_for_network_endpoint_group(name, zone) - logger.info( - 'Loaded NEG "%s" in zone %s', backend.name, backend.zone - ) - self.backends.remove(backend) - self.backend_service_patch_backends() - - def backend_service_patch_backends( - self, max_rate_per_endpoint: Optional[int] = None - ): - logging.info( - "Adding backends to Backend Service %s: %r", - self.backend_service.name, - self.backends, - ) - self.compute.backend_service_patch_backends( - self.backend_service, self.backends, max_rate_per_endpoint - ) - - def backend_service_remove_all_backends(self): - logging.info( - "Removing backends from Backend Service %s", - self.backend_service.name, - ) - self.compute.backend_service_remove_all_backends(self.backend_service) - - def wait_for_backends_healthy_status(self): - logger.info( - "Waiting for Backend Service %s to report all backends healthy: %r", - self.backend_service.name, - [backend.name for backend in self.backends], - ) - self.compute.wait_for_backends_healthy_status( - self.backend_service, self.backends - ) - - def create_alternative_backend_service( - self, protocol: Optional[BackendServiceProtocol] = _BackendGRPC - ): - if protocol is None: - protocol = _BackendGRPC - name = self.make_resource_name(self.ALTERNATIVE_BACKEND_SERVICE_NAME) - logger.info( - 'Creating %s Alternative Backend Service "%s"', protocol.name, name - ) - resource = self.compute.create_backend_service_traffic_director( - name, health_check=self.health_check, protocol=protocol - ) - self.alternative_backend_service = resource - self.alternative_backend_service_protocol = protocol - - def load_alternative_backend_service(self): - name = self.make_resource_name(self.ALTERNATIVE_BACKEND_SERVICE_NAME) - resource = self.compute.get_backend_service_traffic_director(name) - self.alternative_backend_service = resource - - def delete_alternative_backend_service(self, force=False): - if force: - name = self.make_resource_name( - self.ALTERNATIVE_BACKEND_SERVICE_NAME - ) - elif self.alternative_backend_service: - name = self.alternative_backend_service.name - else: - return - logger.info('Deleting Alternative Backend Service "%s"', name) - self.compute.delete_backend_service(name) - self.alternative_backend_service = None - - def alternative_backend_service_add_neg_backends(self, name, zones): - logger.info("Waiting for Network Endpoint Groups to load endpoints.") - for zone in zones: - backend = self.compute.wait_for_network_endpoint_group(name, zone) - logger.info( - 'Loaded NEG "%s" in zone %s', backend.name, backend.zone - ) - self.alternative_backends.add(backend) - self.alternative_backend_service_patch_backends() - - def alternative_backend_service_patch_backends(self): - logging.info( - "Adding backends to Backend Service %s: %r", - self.alternative_backend_service.name, - self.alternative_backends, - ) - self.compute.backend_service_patch_backends( - self.alternative_backend_service, self.alternative_backends - ) - - def alternative_backend_service_remove_all_backends(self): - logging.info( - "Removing backends from Backend Service %s", - self.alternative_backend_service.name, - ) - self.compute.backend_service_remove_all_backends( - self.alternative_backend_service - ) - - def wait_for_alternative_backends_healthy_status(self): - logger.debug( - "Waiting for Backend Service %s to report all backends healthy %r", - self.alternative_backend_service, - self.alternative_backends, - ) - self.compute.wait_for_backends_healthy_status( - self.alternative_backend_service, self.alternative_backends - ) - - def create_affinity_backend_service( - self, protocol: Optional[BackendServiceProtocol] = _BackendGRPC - ): - if protocol is None: - protocol = _BackendGRPC - name = self.make_resource_name(self.AFFINITY_BACKEND_SERVICE_NAME) - logger.info( - 'Creating %s Affinity Backend Service "%s"', protocol.name, name - ) - resource = self.compute.create_backend_service_traffic_director( - name, - health_check=self.health_check, - protocol=protocol, - affinity_header=TEST_AFFINITY_METADATA_KEY, - ) - self.affinity_backend_service = resource - self.affinity_backend_service_protocol = protocol - - def load_affinity_backend_service(self): - name = self.make_resource_name(self.AFFINITY_BACKEND_SERVICE_NAME) - resource = self.compute.get_backend_service_traffic_director(name) - self.affinity_backend_service = resource - - def delete_affinity_backend_service(self, force=False): - if force: - name = self.make_resource_name(self.AFFINITY_BACKEND_SERVICE_NAME) - elif self.affinity_backend_service: - name = self.affinity_backend_service.name - else: - return - logger.info('Deleting Affinity Backend Service "%s"', name) - self.compute.delete_backend_service(name) - self.affinity_backend_service = None - - def affinity_backend_service_add_neg_backends(self, name, zones): - logger.info("Waiting for Network Endpoint Groups to load endpoints.") - for zone in zones: - backend = self.compute.wait_for_network_endpoint_group(name, zone) - logger.info( - 'Loaded NEG "%s" in zone %s', backend.name, backend.zone - ) - self.affinity_backends.add(backend) - self.affinity_backend_service_patch_backends() - - def affinity_backend_service_patch_backends(self): - logging.info( - "Adding backends to Backend Service %s: %r", - self.affinity_backend_service.name, - self.affinity_backends, - ) - self.compute.backend_service_patch_backends( - self.affinity_backend_service, self.affinity_backends - ) - - def affinity_backend_service_remove_all_backends(self): - logging.info( - "Removing backends from Backend Service %s", - self.affinity_backend_service.name, - ) - self.compute.backend_service_remove_all_backends( - self.affinity_backend_service - ) - - def wait_for_affinity_backends_healthy_status(self): - logger.debug( - "Waiting for Backend Service %s to report all backends healthy %r", - self.affinity_backend_service, - self.affinity_backends, - ) - self.compute.wait_for_backends_healthy_status( - self.affinity_backend_service, self.affinity_backends - ) - - @staticmethod - def _generate_url_map_body( - name: str, - matcher_name: str, - src_hosts, - dst_default_backend_service: GcpResource, - dst_host_rule_match_backend_service: Optional[GcpResource] = None, - ) -> Dict[str, Any]: - if dst_host_rule_match_backend_service is None: - dst_host_rule_match_backend_service = dst_default_backend_service - return { - "name": name, - "defaultService": dst_default_backend_service.url, - "hostRules": [ - { - "hosts": src_hosts, - "pathMatcher": matcher_name, - } - ], - "pathMatchers": [ - { - "name": matcher_name, - "defaultService": dst_host_rule_match_backend_service.url, - } - ], - } - - def create_url_map(self, src_host: str, src_port: int) -> GcpResource: - src_address = f"{src_host}:{src_port}" - name = self.make_resource_name(self.URL_MAP_NAME) - matcher_name = self.make_resource_name(self.URL_MAP_PATH_MATCHER_NAME) - logger.info( - 'Creating URL map "%s": %s -> %s', - name, - src_address, - self.backend_service.name, - ) - resource = self.compute.create_url_map_with_content( - self._generate_url_map_body( - name, matcher_name, [src_address], self.backend_service - ) - ) - self.url_map = resource - return resource - - def patch_url_map( - self, src_host: str, src_port: int, backend_service: GcpResource - ): - src_address = f"{src_host}:{src_port}" - name = self.make_resource_name(self.URL_MAP_NAME) - matcher_name = self.make_resource_name(self.URL_MAP_PATH_MATCHER_NAME) - logger.info( - 'Patching URL map "%s": %s -> %s', - name, - src_address, - backend_service.name, - ) - self.compute.patch_url_map( - self.url_map, - self._generate_url_map_body( - name, matcher_name, [src_address], backend_service - ), - ) - - def create_url_map_with_content(self, url_map_body: Any) -> GcpResource: - logger.info("Creating URL map: %s", url_map_body) - resource = self.compute.create_url_map_with_content(url_map_body) - self.url_map = resource - return resource - - def delete_url_map(self, force=False): - if force: - name = self.make_resource_name(self.URL_MAP_NAME) - elif self.url_map: - name = self.url_map.name - else: - return - logger.info('Deleting URL Map "%s"', name) - self.compute.delete_url_map(name) - self.url_map = None - - def create_alternative_url_map( - self, - src_host: str, - src_port: int, - backend_service: Optional[GcpResource] = None, - ) -> GcpResource: - name = self.make_resource_name(self.ALTERNATIVE_URL_MAP_NAME) - src_address = f"{src_host}:{src_port}" - matcher_name = self.make_resource_name(self.URL_MAP_PATH_MATCHER_NAME) - if backend_service is None: - backend_service = self.alternative_backend_service - logger.info( - 'Creating alternative URL map "%s": %s -> %s', - name, - src_address, - backend_service.name, - ) - resource = self.compute.create_url_map_with_content( - self._generate_url_map_body( - name, matcher_name, [src_address], backend_service - ) - ) - self.alternative_url_map = resource - return resource - - def delete_alternative_url_map(self, force=False): - if force: - name = self.make_resource_name(self.ALTERNATIVE_URL_MAP_NAME) - elif self.alternative_url_map: - name = self.alternative_url_map.name - else: - return - logger.info('Deleting alternative URL Map "%s"', name) - self.compute.delete_url_map(name) - self.url_map = None - - def create_target_proxy(self): - name = self.make_resource_name(self.TARGET_PROXY_NAME) - if self.backend_service_protocol is BackendServiceProtocol.GRPC: - target_proxy_type = "GRPC" - create_proxy_fn = self.compute.create_target_grpc_proxy - self.target_proxy_is_http = False - elif self.backend_service_protocol is BackendServiceProtocol.HTTP2: - target_proxy_type = "HTTP" - create_proxy_fn = self.compute.create_target_http_proxy - self.target_proxy_is_http = True - else: - raise TypeError("Unexpected backend service protocol") - - logger.info( - 'Creating target %s proxy "%s" to URL map %s', - name, - target_proxy_type, - self.url_map.name, - ) - self.target_proxy = create_proxy_fn(name, self.url_map) - - def delete_target_grpc_proxy(self, force=False): - if force: - name = self.make_resource_name(self.TARGET_PROXY_NAME) - elif self.target_proxy: - name = self.target_proxy.name - else: - return - logger.info('Deleting Target GRPC proxy "%s"', name) - self.compute.delete_target_grpc_proxy(name) - self.target_proxy = None - self.target_proxy_is_http = False - - def delete_target_http_proxy(self, force=False): - if force: - name = self.make_resource_name(self.TARGET_PROXY_NAME) - elif self.target_proxy and self.target_proxy_is_http: - name = self.target_proxy.name - else: - return - logger.info('Deleting HTTP Target proxy "%s"', name) - self.compute.delete_target_http_proxy(name) - self.target_proxy = None - self.target_proxy_is_http = False - - def create_alternative_target_proxy(self): - name = self.make_resource_name(self.ALTERNATIVE_TARGET_PROXY_NAME) - if self.backend_service_protocol is BackendServiceProtocol.GRPC: - logger.info( - 'Creating alternative target GRPC proxy "%s" to URL map %s', - name, - self.alternative_url_map.name, - ) - self.alternative_target_proxy = ( - self.compute.create_target_grpc_proxy( - name, self.alternative_url_map, False - ) - ) - else: - raise TypeError("Unexpected backend service protocol") - - def delete_alternative_target_grpc_proxy(self, force=False): - if force: - name = self.make_resource_name(self.ALTERNATIVE_TARGET_PROXY_NAME) - elif self.alternative_target_proxy: - name = self.alternative_target_proxy.name - else: - return - logger.info('Deleting alternative Target GRPC proxy "%s"', name) - self.compute.delete_target_grpc_proxy(name) - self.alternative_target_proxy = None - - def find_unused_forwarding_rule_port( - self, - *, - lo: int = 1024, # To avoid confusion, skip well-known ports. - hi: int = 65535, - attempts: int = 25, - ) -> int: - for _ in range(attempts): - src_port = random.randint(lo, hi) - if not self.compute.exists_forwarding_rule(src_port): - return src_port - # TODO(sergiitk): custom exception - raise RuntimeError("Couldn't find unused forwarding rule port") - - def create_forwarding_rule(self, src_port: int): - name = self.make_resource_name(self.FORWARDING_RULE_NAME) - src_port = int(src_port) - logging.info( - 'Creating forwarding rule "%s" in network "%s": 0.0.0.0:%s -> %s', - name, - self.network, - src_port, - self.target_proxy.url, - ) - resource = self.compute.create_forwarding_rule( - name, src_port, self.target_proxy, self.network_url - ) - self.forwarding_rule = resource - return resource - - def delete_forwarding_rule(self, force=False): - if force: - name = self.make_resource_name(self.FORWARDING_RULE_NAME) - elif self.forwarding_rule: - name = self.forwarding_rule.name - else: - return - logger.info('Deleting Forwarding rule "%s"', name) - self.compute.delete_forwarding_rule(name) - self.forwarding_rule = None - - def create_alternative_forwarding_rule( - self, src_port: int, ip_address="0.0.0.0" - ): - name = self.make_resource_name(self.ALTERNATIVE_FORWARDING_RULE_NAME) - src_port = int(src_port) - logging.info( - ( - 'Creating alternative forwarding rule "%s" in network "%s":' - " %s:%s -> %s" - ), - name, - self.network, - ip_address, - src_port, - self.alternative_target_proxy.url, - ) - resource = self.compute.create_forwarding_rule( - name, - src_port, - self.alternative_target_proxy, - self.network_url, - ip_address=ip_address, - ) - self.alternative_forwarding_rule = resource - return resource - - def delete_alternative_forwarding_rule(self, force=False): - if force: - name = self.make_resource_name( - self.ALTERNATIVE_FORWARDING_RULE_NAME - ) - elif self.alternative_forwarding_rule: - name = self.alternative_forwarding_rule.name - else: - return - logger.info('Deleting alternative Forwarding rule "%s"', name) - self.compute.delete_forwarding_rule(name) - self.alternative_forwarding_rule = None - - def create_firewall_rule(self, allowed_ports: List[str]): - name = self.make_resource_name(self.FIREWALL_RULE_NAME) - logging.info( - 'Creating firewall rule "%s" in network "%s" with allowed ports %s', - name, - self.network, - allowed_ports, - ) - resource = self.compute.create_firewall_rule( - name, - self.network_url, - xds_flags.FIREWALL_SOURCE_RANGE.value, - allowed_ports, - ) - self.firewall_rule = resource - - def delete_firewall_rule(self, force=False): - """The firewall rule won't be automatically removed.""" - if force: - name = self.make_resource_name(self.FIREWALL_RULE_NAME) - elif self.firewall_rule: - name = self.firewall_rule.name - else: - return - logger.info('Deleting Firewall Rule "%s"', name) - self.compute.delete_firewall_rule(name) - self.firewall_rule = None - - -class TrafficDirectorAppNetManager(TrafficDirectorManager): - GRPC_ROUTE_NAME = "grpc-route" - HTTP_ROUTE_NAME = "http-route" - MESH_NAME = "mesh" - - netsvc: _NetworkServicesV1Alpha1 - - def __init__( - self, - gcp_api_manager: gcp.api.GcpApiManager, - project: str, - *, - resource_prefix: str, - resource_suffix: Optional[str] = None, - network: str = "default", - compute_api_version: str = "v1", - ): - super().__init__( - gcp_api_manager, - project, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - network=network, - compute_api_version=compute_api_version, - ) - - # API - self.netsvc = _NetworkServicesV1Alpha1(gcp_api_manager, project) - - # Managed resources - # TODO(gnossen) PTAL at the pylint error - self.grpc_route: Optional[GrpcRoute] = None - self.http_route: Optional[HttpRoute] = None - self.mesh: Optional[Mesh] = None - - def create_mesh(self) -> GcpResource: - name = self.make_resource_name(self.MESH_NAME) - logger.info("Creating Mesh %s", name) - body = {} - resource = self.netsvc.create_mesh(name, body) - self.mesh = self.netsvc.get_mesh(name) - logger.debug("Loaded Mesh: %s", self.mesh) - return resource - - def delete_mesh(self, force=False): - if force: - name = self.make_resource_name(self.MESH_NAME) - elif self.mesh: - name = self.mesh.name - else: - return - logger.info("Deleting Mesh %s", name) - self.netsvc.delete_mesh(name) - self.mesh = None - - def create_grpc_route(self, src_host: str, src_port: int) -> GcpResource: - host = f"{src_host}:{src_port}" - service_name = self.netsvc.resource_full_name( - self.backend_service.name, "backendServices" - ) - body = { - "meshes": [self.mesh.url], - "hostnames": host, - "rules": [ - {"action": {"destinations": [{"serviceName": service_name}]}} - ], - } - name = self.make_resource_name(self.GRPC_ROUTE_NAME) - logger.info("Creating GrpcRoute %s", name) - resource = self.netsvc.create_grpc_route(name, body) - self.grpc_route = self.netsvc.get_grpc_route(name) - logger.debug("Loaded GrpcRoute: %s", self.grpc_route) - return resource - - def create_grpc_route_with_content(self, body: Any) -> GcpResource: - name = self.make_resource_name(self.GRPC_ROUTE_NAME) - logger.info("Creating GrpcRoute %s", name) - resource = self.netsvc.create_grpc_route(name, body) - self.grpc_route = self.netsvc.get_grpc_route(name) - logger.debug("Loaded GrpcRoute: %s", self.grpc_route) - return resource - - def create_http_route_with_content(self, body: Any) -> GcpResource: - name = self.make_resource_name(self.HTTP_ROUTE_NAME) - logger.info("Creating HttpRoute %s", name) - resource = self.netsvc.create_http_route(name, body) - self.http_route = self.netsvc.get_http_route(name) - logger.debug("Loaded HttpRoute: %s", self.http_route) - return resource - - def delete_grpc_route(self, force=False): - if force: - name = self.make_resource_name(self.GRPC_ROUTE_NAME) - elif self.grpc_route: - name = self.grpc_route.name - else: - return - logger.info("Deleting GrpcRoute %s", name) - self.netsvc.delete_grpc_route(name) - self.grpc_route = None - - def delete_http_route(self, force=False): - if force: - name = self.make_resource_name(self.HTTP_ROUTE_NAME) - elif self.http_route: - name = self.http_route.name - else: - return - logger.info("Deleting HttpRoute %s", name) - self.netsvc.delete_http_route(name) - self.http_route = None - - def cleanup(self, *, force=False): - self.delete_http_route(force=force) - self.delete_grpc_route(force=force) - self.delete_mesh(force=force) - super().cleanup(force=force) - - -class TrafficDirectorSecureManager(TrafficDirectorManager): - SERVER_TLS_POLICY_NAME = "server-tls-policy" - CLIENT_TLS_POLICY_NAME = "client-tls-policy" - AUTHZ_POLICY_NAME = "authz-policy" - ENDPOINT_POLICY = "endpoint-policy" - CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" - - netsec: _NetworkSecurityV1Beta1 - netsvc: _NetworkServicesV1Beta1 - - def __init__( - self, - gcp_api_manager: gcp.api.GcpApiManager, - project: str, - *, - resource_prefix: str, - resource_suffix: Optional[str] = None, - network: str = "default", - compute_api_version: str = "v1", - ): - super().__init__( - gcp_api_manager, - project, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - network=network, - compute_api_version=compute_api_version, - ) - - # API - self.netsec = _NetworkSecurityV1Beta1(gcp_api_manager, project) - self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) - - # Managed resources - self.server_tls_policy: Optional[ServerTlsPolicy] = None - self.client_tls_policy: Optional[ClientTlsPolicy] = None - self.authz_policy: Optional[AuthorizationPolicy] = None - self.endpoint_policy: Optional[EndpointPolicy] = None - - def setup_server_security( - self, *, server_namespace, server_name, server_port, tls=True, mtls=True - ): - self.create_server_tls_policy(tls=tls, mtls=mtls) - self.create_endpoint_policy( - server_namespace=server_namespace, - server_name=server_name, - server_port=server_port, - ) - - def setup_client_security( - self, *, server_namespace, server_name, tls=True, mtls=True - ): - self.create_client_tls_policy(tls=tls, mtls=mtls) - self.backend_service_apply_client_mtls_policy( - server_namespace, server_name - ) - - def cleanup(self, *, force=False): - # Cleanup in the reverse order of creation - super().cleanup(force=force) - self.delete_endpoint_policy(force=force) - self.delete_server_tls_policy(force=force) - self.delete_client_tls_policy(force=force) - self.delete_authz_policy(force=force) - - def create_server_tls_policy(self, *, tls, mtls): - name = self.make_resource_name(self.SERVER_TLS_POLICY_NAME) - logger.info("Creating Server TLS Policy %s", name) - if not tls and not mtls: - logger.warning( - ( - "Server TLS Policy %s neither TLS, nor mTLS " - "policy. Skipping creation" - ), - name, - ) - return - - certificate_provider = self._get_certificate_provider() - policy = {} - if tls: - policy["serverCertificate"] = certificate_provider - if mtls: - policy["mtlsPolicy"] = { - "clientValidationCa": [certificate_provider], - } - - self.netsec.create_server_tls_policy(name, policy) - self.server_tls_policy = self.netsec.get_server_tls_policy(name) - logger.debug("Server TLS Policy loaded: %r", self.server_tls_policy) - - def delete_server_tls_policy(self, force=False): - if force: - name = self.make_resource_name(self.SERVER_TLS_POLICY_NAME) - elif self.server_tls_policy: - name = self.server_tls_policy.name - else: - return - logger.info("Deleting Server TLS Policy %s", name) - self.netsec.delete_server_tls_policy(name) - self.server_tls_policy = None - - def create_authz_policy(self, *, action: str, rules: list): - name = self.make_resource_name(self.AUTHZ_POLICY_NAME) - logger.info("Creating Authz Policy %s", name) - policy = { - "action": action, - "rules": rules, - } - - self.netsec.create_authz_policy(name, policy) - self.authz_policy = self.netsec.get_authz_policy(name) - logger.debug("Authz Policy loaded: %r", self.authz_policy) - - def delete_authz_policy(self, force=False): - if force: - name = self.make_resource_name(self.AUTHZ_POLICY_NAME) - elif self.authz_policy: - name = self.authz_policy.name - else: - return - logger.info("Deleting Authz Policy %s", name) - self.netsec.delete_authz_policy(name) - self.authz_policy = None - - def create_endpoint_policy( - self, *, server_namespace: str, server_name: str, server_port: int - ) -> None: - name = self.make_resource_name(self.ENDPOINT_POLICY) - logger.info("Creating Endpoint Policy %s", name) - endpoint_matcher_labels = [ - { - "labelName": "app", - "labelValue": f"{server_namespace}-{server_name}", - } - ] - port_selector = {"ports": [str(server_port)]} - label_matcher_all = { - "metadataLabelMatchCriteria": "MATCH_ALL", - "metadataLabels": endpoint_matcher_labels, - } - config = { - "type": "GRPC_SERVER", - "trafficPortSelector": port_selector, - "endpointMatcher": { - "metadataLabelMatcher": label_matcher_all, - }, - } - if self.server_tls_policy: - config["serverTlsPolicy"] = self.server_tls_policy.name - else: - logger.warning( - ( - "Creating Endpoint Policy %s with " - "no Server TLS policy attached" - ), - name, - ) - if self.authz_policy: - config["authorizationPolicy"] = self.authz_policy.name - - self.netsvc.create_endpoint_policy(name, config) - self.endpoint_policy = self.netsvc.get_endpoint_policy(name) - logger.debug("Loaded Endpoint Policy: %r", self.endpoint_policy) - - def delete_endpoint_policy(self, force: bool = False) -> None: - if force: - name = self.make_resource_name(self.ENDPOINT_POLICY) - elif self.endpoint_policy: - name = self.endpoint_policy.name - else: - return - logger.info("Deleting Endpoint Policy %s", name) - self.netsvc.delete_endpoint_policy(name) - self.endpoint_policy = None - - def create_client_tls_policy(self, *, tls, mtls): - name = self.make_resource_name(self.CLIENT_TLS_POLICY_NAME) - logger.info("Creating Client TLS Policy %s", name) - if not tls and not mtls: - logger.warning( - ( - "Client TLS Policy %s neither TLS, nor mTLS " - "policy. Skipping creation" - ), - name, - ) - return - - certificate_provider = self._get_certificate_provider() - policy = {} - if tls: - policy["serverValidationCa"] = [certificate_provider] - if mtls: - policy["clientCertificate"] = certificate_provider - - self.netsec.create_client_tls_policy(name, policy) - self.client_tls_policy = self.netsec.get_client_tls_policy(name) - logger.debug("Client TLS Policy loaded: %r", self.client_tls_policy) - - def delete_client_tls_policy(self, force=False): - if force: - name = self.make_resource_name(self.CLIENT_TLS_POLICY_NAME) - elif self.client_tls_policy: - name = self.client_tls_policy.name - else: - return - logger.info("Deleting Client TLS Policy %s", name) - self.netsec.delete_client_tls_policy(name) - self.client_tls_policy = None - - def backend_service_apply_client_mtls_policy( - self, - server_namespace, - server_name, - ): - if not self.client_tls_policy: - logger.warning( - ( - "Client TLS policy not created, " - "skipping attaching to Backend Service %s" - ), - self.backend_service.name, - ) - return - - server_spiffe = ( - f"spiffe://{self.project}.svc.id.goog/" - f"ns/{server_namespace}/sa/{server_name}" - ) - logging.info( - "Adding Client TLS Policy to Backend Service %s: %s, server %s", - self.backend_service.name, - self.client_tls_policy.url, - server_spiffe, - ) - - self.compute.patch_backend_service( - self.backend_service, - { - "securitySettings": { - "clientTlsPolicy": self.client_tls_policy.url, - "subjectAltNames": [server_spiffe], - } - }, - ) - - @classmethod - def _get_certificate_provider(cls): - return { - "certificateProviderInstance": { - "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, - }, - } diff --git a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director_gamma.py b/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director_gamma.py deleted file mode 100644 index 77c5f9cdc3d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/infrastructure/traffic_director_gamma.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import framework.infrastructure.traffic_director as td_base - - -# TODO(sergiitk): [GAMMA] make a TD-manager-less base test case. -class TrafficDirectorGammaManager(td_base.TrafficDirectorManager): - """Gamma.""" - - def cleanup(self, *, force=False): # pylint: disable=unused-argument - return True diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/__init__.py deleted file mode 100644 index 30d226342a1..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from framework.rpc import grpc diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc.py deleted file mode 100644 index b67991115be..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import re -from typing import Any, Dict, Optional - -from google.protobuf import json_format -import google.protobuf.message -import grpc - -import framework.errors - -logger = logging.getLogger(__name__) - -# Type aliases -Message = google.protobuf.message.Message -RpcError = grpc.RpcError - - -class GrpcClientHelper: - DEFAULT_RPC_DEADLINE_SEC = 90 - channel: grpc.Channel - # This is purely cosmetic to make RPC logs look like method calls. - log_service_name: str - # This is purely cosmetic to output the RPC target. Normally set to the - # hostname:port of the remote service, but it doesn't have to be the - # real target. This is done so that when RPC are routed to the proxy - # or port forwarding, this still is set to a useful name. - log_target: str - - def __init__( - self, - channel: grpc.Channel, - stub_class: Any, - *, - log_target: Optional[str] = "", - ): - self.channel = channel - self.stub = stub_class(channel) - self.log_service_name = re.sub( - "Stub$", "", self.stub.__class__.__name__ - ) - self.log_target = log_target or "" - - def call_unary_with_deadline( - self, - *, - rpc: str, - req: Message, - deadline_sec: Optional[int] = DEFAULT_RPC_DEADLINE_SEC, - log_level: Optional[int] = logging.DEBUG, - ) -> Message: - if deadline_sec is None: - deadline_sec = self.DEFAULT_RPC_DEADLINE_SEC - - call_kwargs = dict(wait_for_ready=True, timeout=deadline_sec) - self._log_rpc_request(rpc, req, call_kwargs, log_level) - - # Call RPC, e.g. RpcStub(channel).RpcMethod(req, ...options) - rpc_callable: grpc.UnaryUnaryMultiCallable = getattr(self.stub, rpc) - return rpc_callable(req, **call_kwargs) - - def _log_rpc_request(self, rpc, req, call_kwargs, log_level=logging.DEBUG): - logger.log( - logging.DEBUG if log_level is None else log_level, - "[%s] >> RPC %s.%s(request=%s(%r), %s)", - self.log_target, - self.log_service_name, - rpc, - req.__class__.__name__, - json_format.MessageToDict(req), - ", ".join({f"{k}={v}" for k, v in call_kwargs.items()}), - ) - - -class GrpcApp: - channels: Dict[int, grpc.Channel] - - class NotFound(framework.errors.FrameworkError): - """Requested resource not found""" - - def __init__(self, rpc_host): - self.rpc_host = rpc_host - # Cache gRPC channels per port - self.channels = dict() - - def _make_channel(self, port) -> grpc.Channel: - if port not in self.channels: - target = f"{self.rpc_host}:{port}" - self.channels[port] = grpc.insecure_channel(target) - return self.channels[port] - - def close(self): - # Close all channels - for channel in self.channels.values(): - channel.close() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - return False - - def __del__(self): - self.close() diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_channelz.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_channelz.py deleted file mode 100644 index df25e2e5aaf..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_channelz.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This contains helpers for gRPC services defined in -https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto -""" -import ipaddress -import logging -from typing import Iterator, Optional - -import grpc -from grpc_channelz.v1 import channelz_pb2 -from grpc_channelz.v1 import channelz_pb2_grpc - -import framework.rpc - -logger = logging.getLogger(__name__) - -# Type aliases -# Channel -Channel = channelz_pb2.Channel -ChannelData = channelz_pb2.ChannelData -ChannelConnectivityState = channelz_pb2.ChannelConnectivityState -ChannelState = ChannelConnectivityState.State # pylint: disable=no-member -_GetTopChannelsRequest = channelz_pb2.GetTopChannelsRequest -_GetTopChannelsResponse = channelz_pb2.GetTopChannelsResponse -# Subchannel -Subchannel = channelz_pb2.Subchannel -_GetSubchannelRequest = channelz_pb2.GetSubchannelRequest -_GetSubchannelResponse = channelz_pb2.GetSubchannelResponse -# Server -Server = channelz_pb2.Server -_GetServersRequest = channelz_pb2.GetServersRequest -_GetServersResponse = channelz_pb2.GetServersResponse -# Sockets -Socket = channelz_pb2.Socket -SocketRef = channelz_pb2.SocketRef -_GetSocketRequest = channelz_pb2.GetSocketRequest -_GetSocketResponse = channelz_pb2.GetSocketResponse -Address = channelz_pb2.Address -Security = channelz_pb2.Security -# Server Sockets -_GetServerSocketsRequest = channelz_pb2.GetServerSocketsRequest -_GetServerSocketsResponse = channelz_pb2.GetServerSocketsResponse - - -class ChannelzServiceClient(framework.rpc.grpc.GrpcClientHelper): - stub: channelz_pb2_grpc.ChannelzStub - - def __init__( - self, channel: grpc.Channel, *, log_target: Optional[str] = "" - ): - super().__init__( - channel, channelz_pb2_grpc.ChannelzStub, log_target=log_target - ) - - @staticmethod - def is_sock_tcpip_address(address: Address): - return address.WhichOneof("address") == "tcpip_address" - - @staticmethod - def is_ipv4(tcpip_address: Address.TcpIpAddress): - # According to proto, tcpip_address.ip_address is either IPv4 or IPv6. - # Correspondingly, it's either 4 bytes or 16 bytes in length. - return len(tcpip_address.ip_address) == 4 - - @classmethod - def sock_address_to_str(cls, address: Address): - if cls.is_sock_tcpip_address(address): - tcpip_address: Address.TcpIpAddress = address.tcpip_address - if cls.is_ipv4(tcpip_address): - ip = ipaddress.IPv4Address(tcpip_address.ip_address) - else: - ip = ipaddress.IPv6Address(tcpip_address.ip_address) - return f"{ip}:{tcpip_address.port}" - else: - raise NotImplementedError("Only tcpip_address implemented") - - @classmethod - def sock_addresses_pretty(cls, socket: Socket): - return ( - f"local={cls.sock_address_to_str(socket.local)}, " - f"remote={cls.sock_address_to_str(socket.remote)}" - ) - - @staticmethod - def find_server_socket_matching_client( - server_sockets: Iterator[Socket], client_socket: Socket - ) -> Socket: - for server_socket in server_sockets: - if server_socket.remote == client_socket.local: - return server_socket - return None - - @staticmethod - def channel_repr(channel: Channel) -> str: - result = f"" - return result - - @staticmethod - def subchannel_repr(subchannel: Subchannel) -> str: - result = f"" - return result - - def find_channels_for_target( - self, target: str, **kwargs - ) -> Iterator[Channel]: - return ( - channel - for channel in self.list_channels(**kwargs) - if channel.data.target == target - ) - - def find_server_listening_on_port( - self, port: int, **kwargs - ) -> Optional[Server]: - for server in self.list_servers(**kwargs): - listen_socket_ref: SocketRef - for listen_socket_ref in server.listen_socket: - listen_socket = self.get_socket( - listen_socket_ref.socket_id, **kwargs - ) - listen_address: Address = listen_socket.local - if ( - self.is_sock_tcpip_address(listen_address) - and listen_address.tcpip_address.port == port - ): - return server - return None - - def list_channels(self, **kwargs) -> Iterator[Channel]: - """ - Iterate over all pages of all root channels. - - Root channels are those which application has directly created. - This does not include subchannels nor non-top level channels. - """ - start: int = -1 - response: Optional[_GetTopChannelsResponse] = None - while start < 0 or not response.end: - # From proto: To request subsequent pages, the client generates this - # value by adding 1 to the highest seen result ID. - start += 1 - response = self.call_unary_with_deadline( - rpc="GetTopChannels", - req=_GetTopChannelsRequest(start_channel_id=start), - **kwargs, - ) - for channel in response.channel: - start = max(start, channel.ref.channel_id) - yield channel - - def get_channel(self, channel_id, **kwargs) -> Channel: - """Return a single Channel, otherwise raises RpcError.""" - response: channelz_pb2.GetChannelResponse - try: - response = self.call_unary_with_deadline( - rpc="GetChannel", - req=channelz_pb2.GetChannelRequest(channel_id=channel_id), - **kwargs, - ) - return response.channel - except grpc.RpcError as err: - if isinstance(err, grpc.Call): - # Translate NOT_FOUND into GrpcApp.NotFound. - if err.code() is grpc.StatusCode.NOT_FOUND: - raise framework.rpc.grpc.GrpcApp.NotFound( - f"Channel with channel_id {channel_id} not found", - ) - - raise - - def list_servers(self, **kwargs) -> Iterator[Server]: - """Iterate over all pages of all servers that exist in the process.""" - start: int = -1 - response: Optional[_GetServersResponse] = None - while start < 0 or not response.end: - # From proto: To request subsequent pages, the client generates this - # value by adding 1 to the highest seen result ID. - start += 1 - response = self.call_unary_with_deadline( - rpc="GetServers", - req=_GetServersRequest(start_server_id=start), - **kwargs, - ) - for server in response.server: - start = max(start, server.ref.server_id) - yield server - - def list_server_sockets(self, server: Server, **kwargs) -> Iterator[Socket]: - """List all server sockets that exist in server process. - - Iterating over the results will resolve additional pages automatically. - """ - start: int = -1 - response: Optional[_GetServerSocketsResponse] = None - while start < 0 or not response.end: - # From proto: To request subsequent pages, the client generates this - # value by adding 1 to the highest seen result ID. - start += 1 - response = self.call_unary_with_deadline( - rpc="GetServerSockets", - req=_GetServerSocketsRequest( - server_id=server.ref.server_id, start_socket_id=start - ), - **kwargs, - ) - socket_ref: SocketRef - for socket_ref in response.socket_ref: - start = max(start, socket_ref.socket_id) - # Yield actual socket - yield self.get_socket(socket_ref.socket_id, **kwargs) - - def list_channel_sockets( - self, channel: Channel, **kwargs - ) -> Iterator[Socket]: - """List all sockets of all subchannels of a given channel.""" - for subchannel in self.list_channel_subchannels(channel, **kwargs): - yield from self.list_subchannels_sockets(subchannel, **kwargs) - - def list_channel_subchannels( - self, channel: Channel, **kwargs - ) -> Iterator[Subchannel]: - """List all subchannels of a given channel.""" - for subchannel_ref in channel.subchannel_ref: - yield self.get_subchannel(subchannel_ref.subchannel_id, **kwargs) - - def list_subchannels_sockets( - self, subchannel: Subchannel, **kwargs - ) -> Iterator[Socket]: - """List all sockets of a given subchannel.""" - for socket_ref in subchannel.socket_ref: - yield self.get_socket(socket_ref.socket_id, **kwargs) - - def get_subchannel(self, subchannel_id, **kwargs) -> Subchannel: - """Return a single Subchannel, otherwise raises RpcError.""" - response: _GetSubchannelResponse = self.call_unary_with_deadline( - rpc="GetSubchannel", - req=_GetSubchannelRequest(subchannel_id=subchannel_id), - **kwargs, - ) - return response.subchannel - - def get_socket(self, socket_id, **kwargs) -> Socket: - """Return a single Socket, otherwise raises RpcError.""" - response: _GetSocketResponse = self.call_unary_with_deadline( - rpc="GetSocket", - req=_GetSocketRequest(socket_id=socket_id), - **kwargs, - ) - return response.socket diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_csds.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_csds.py deleted file mode 100644 index 03fa6328c23..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_csds.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This contains helpers for gRPC services defined in -https://github.com/envoyproxy/envoy/blob/main/api/envoy/service/status/v3/csds.proto -""" - -import logging -from typing import Optional - -# Needed to load the descriptors so that Any is parsed -# TODO(sergiitk): replace with import xds_protos when it works -# isort: off -# pylint: disable=unused-import,ungrouped-imports -import framework.rpc.xds_protos_imports - -# pylint: enable=unused-import,ungrouped-imports -# isort: on - -from envoy.service.status.v3 import csds_pb2 -from envoy.service.status.v3 import csds_pb2_grpc -import grpc - -import framework.rpc - -logger = logging.getLogger(__name__) - -# Type aliases -ClientConfig = csds_pb2.ClientConfig -_ClientStatusRequest = csds_pb2.ClientStatusRequest - - -class CsdsClient(framework.rpc.grpc.GrpcClientHelper): - stub: csds_pb2_grpc.ClientStatusDiscoveryServiceStub - - def __init__( - self, channel: grpc.Channel, *, log_target: Optional[str] = "" - ): - super().__init__( - channel, - csds_pb2_grpc.ClientStatusDiscoveryServiceStub, - log_target=log_target, - ) - - def fetch_client_status(self, **kwargs) -> Optional[ClientConfig]: - """Fetches the active xDS configurations.""" - response = self.call_unary_with_deadline( - rpc="FetchClientStatus", req=_ClientStatusRequest(), **kwargs - ) - if len(response.config) != 1: - logger.debug( - "Unexpected number of client configs: %s", len(response.config) - ) - return None - return response.config[0] diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_testing.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_testing.py deleted file mode 100644 index a3f0031a492..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/grpc_testing.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This contains helpers for gRPC services defined in -https://github.com/grpc/grpc/blob/master/src/proto/grpc/testing/test.proto -""" -import logging -from typing import Iterable, Optional, Tuple - -import grpc -from grpc_health.v1 import health_pb2 -from grpc_health.v1 import health_pb2_grpc - -import framework.rpc -from src.proto.grpc.testing import empty_pb2 -from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import test_pb2_grpc - -# Type aliases -_LoadBalancerStatsRequest = messages_pb2.LoadBalancerStatsRequest -LoadBalancerStatsResponse = messages_pb2.LoadBalancerStatsResponse -_LoadBalancerAccumulatedStatsRequest = ( - messages_pb2.LoadBalancerAccumulatedStatsRequest -) -LoadBalancerAccumulatedStatsResponse = ( - messages_pb2.LoadBalancerAccumulatedStatsResponse -) -MethodStats = messages_pb2.LoadBalancerAccumulatedStatsResponse.MethodStats -RpcsByPeer = messages_pb2.LoadBalancerStatsResponse.RpcsByPeer - - -class LoadBalancerStatsServiceClient(framework.rpc.grpc.GrpcClientHelper): - stub: test_pb2_grpc.LoadBalancerStatsServiceStub - STATS_PARTIAL_RESULTS_TIMEOUT_SEC = 1200 - STATS_ACCUMULATED_RESULTS_TIMEOUT_SEC = 600 - - def __init__( - self, channel: grpc.Channel, *, log_target: Optional[str] = "" - ): - super().__init__( - channel, - test_pb2_grpc.LoadBalancerStatsServiceStub, - log_target=log_target, - ) - - def get_client_stats( - self, - *, - num_rpcs: int, - timeout_sec: Optional[int] = STATS_PARTIAL_RESULTS_TIMEOUT_SEC, - metadata_keys: Optional[tuple[str, ...]] = None, - ) -> LoadBalancerStatsResponse: - if timeout_sec is None: - timeout_sec = self.STATS_PARTIAL_RESULTS_TIMEOUT_SEC - - return self.call_unary_with_deadline( - rpc="GetClientStats", - req=_LoadBalancerStatsRequest( - num_rpcs=num_rpcs, - timeout_sec=timeout_sec, - metadata_keys=metadata_keys or None, - ), - deadline_sec=timeout_sec, - log_level=logging.INFO, - ) - - def get_client_accumulated_stats( - self, *, timeout_sec: Optional[int] = None - ) -> LoadBalancerAccumulatedStatsResponse: - if timeout_sec is None: - timeout_sec = self.STATS_ACCUMULATED_RESULTS_TIMEOUT_SEC - - return self.call_unary_with_deadline( - rpc="GetClientAccumulatedStats", - req=_LoadBalancerAccumulatedStatsRequest(), - deadline_sec=timeout_sec, - log_level=logging.INFO, - ) - - -class XdsUpdateClientConfigureServiceClient( - framework.rpc.grpc.GrpcClientHelper -): - stub: test_pb2_grpc.XdsUpdateClientConfigureServiceStub - CONFIGURE_TIMEOUT_SEC: int = 5 - - def __init__( - self, channel: grpc.Channel, *, log_target: Optional[str] = "" - ): - super().__init__( - channel, - test_pb2_grpc.XdsUpdateClientConfigureServiceStub, - log_target=log_target, - ) - - def configure( - self, - *, - rpc_types: Iterable[str], - metadata: Optional[Iterable[Tuple[str, str, str]]] = None, - app_timeout: Optional[int] = None, - timeout_sec: int = CONFIGURE_TIMEOUT_SEC, - ) -> None: - request = messages_pb2.ClientConfigureRequest() - for rpc_type in rpc_types: - request.types.append( - messages_pb2.ClientConfigureRequest.RpcType.Value(rpc_type) - ) - if metadata: - for entry in metadata: - request.metadata.append( - messages_pb2.ClientConfigureRequest.Metadata( - type=messages_pb2.ClientConfigureRequest.RpcType.Value( - entry[0] - ), - key=entry[1], - value=entry[2], - ) - ) - if app_timeout: - request.timeout_sec = app_timeout - # Configure's response is empty - self.call_unary_with_deadline( - rpc="Configure", - req=request, - deadline_sec=timeout_sec, - log_level=logging.INFO, - ) - - -class XdsUpdateHealthServiceClient(framework.rpc.grpc.GrpcClientHelper): - stub: test_pb2_grpc.XdsUpdateHealthServiceStub - - def __init__(self, channel: grpc.Channel, log_target: Optional[str] = ""): - super().__init__( - channel, - test_pb2_grpc.XdsUpdateHealthServiceStub, - log_target=log_target, - ) - - def set_serving(self): - self.call_unary_with_deadline( - rpc="SetServing", req=empty_pb2.Empty(), log_level=logging.INFO - ) - - def set_not_serving(self): - self.call_unary_with_deadline( - rpc="SetNotServing", req=empty_pb2.Empty(), log_level=logging.INFO - ) - - -class HealthClient(framework.rpc.grpc.GrpcClientHelper): - stub: health_pb2_grpc.HealthStub - - def __init__(self, channel: grpc.Channel, log_target: Optional[str] = ""): - super().__init__( - channel, health_pb2_grpc.HealthStub, log_target=log_target - ) - - def check_health(self): - return self.call_unary_with_deadline( - rpc="Check", - req=health_pb2.HealthCheckRequest(), - log_level=logging.INFO, - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/rpc/xds_protos_imports.py b/tools/run_tests/xds_k8s_test_driver/framework/rpc/xds_protos_imports.py deleted file mode 100644 index 5b151bb5020..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/rpc/xds_protos_imports.py +++ /dev/null @@ -1,849 +0,0 @@ -# Copyright 2023 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Envoy protos provided by PyPI package xds-protos""" -# pylint: disable=unused-import,ungrouped-imports,reimported -# isort: off - -from envoy.config.health_checker.redis.v2 import redis_pb2 -from envoy.config.listener.v3 import listener_components_pb2 -from envoy.config.listener.v3 import udp_listener_config_pb2 -from envoy.config.listener.v3 import quic_config_pb2 -from envoy.config.listener.v3 import api_listener_pb2 -from envoy.config.listener.v3 import listener_pb2 -from envoy.config.listener.v2 import api_listener_pb2 -from envoy.config.transport_socket.alts.v2alpha import alts_pb2 -from envoy.config.transport_socket.raw_buffer.v2 import raw_buffer_pb2 -from envoy.config.transport_socket.tap.v2alpha import tap_pb2 -from envoy.config.core.v3 import base_pb2 -from envoy.config.core.v3 import substitution_format_string_pb2 -from envoy.config.core.v3 import backoff_pb2 -from envoy.config.core.v3 import grpc_service_pb2 -from envoy.config.core.v3 import proxy_protocol_pb2 -from envoy.config.core.v3 import protocol_pb2 -from envoy.config.core.v3 import address_pb2 -from envoy.config.core.v3 import health_check_pb2 -from envoy.config.core.v3 import udp_socket_config_pb2 -from envoy.config.core.v3 import grpc_method_list_pb2 -from envoy.config.core.v3 import socket_option_pb2 -from envoy.config.core.v3 import extension_pb2 -from envoy.config.core.v3 import config_source_pb2 -from envoy.config.core.v3 import event_service_config_pb2 -from envoy.config.core.v3 import http_uri_pb2 -from envoy.config.core.v3 import resolver_pb2 -from envoy.config.retry.previous_hosts.v2 import previous_hosts_pb2 -from envoy.config.retry.previous_priorities import ( - previous_priorities_config_pb2, -) -from envoy.config.retry.omit_host_metadata.v2 import ( - omit_host_metadata_config_pb2, -) -from envoy.config.retry.omit_canary_hosts.v2 import omit_canary_hosts_pb2 -from envoy.config.common.dynamic_forward_proxy.v2alpha import dns_cache_pb2 -from envoy.config.common.mutation_rules.v3 import mutation_rules_pb2 -from envoy.config.common.matcher.v3 import matcher_pb2 -from envoy.config.common.tap.v2alpha import common_pb2 -from envoy.config.common.key_value.v3 import config_pb2 -from envoy.config.metrics.v3 import stats_pb2 -from envoy.config.metrics.v3 import metrics_service_pb2 -from envoy.config.metrics.v2 import stats_pb2 -from envoy.config.metrics.v2 import metrics_service_pb2 -from envoy.config.ratelimit.v3 import rls_pb2 -from envoy.config.ratelimit.v2 import rls_pb2 -from envoy.config.trace.v2alpha import xray_pb2 -from envoy.config.trace.v3 import http_tracer_pb2 -from envoy.config.trace.v3 import zipkin_pb2 -from envoy.config.trace.v3 import trace_pb2 -from envoy.config.trace.v3 import lightstep_pb2 -from envoy.config.trace.v3 import datadog_pb2 -from envoy.config.trace.v3 import opentelemetry_pb2 -from envoy.config.trace.v3 import opencensus_pb2 -from envoy.config.trace.v3 import dynamic_ot_pb2 -from envoy.config.trace.v3 import xray_pb2 -from envoy.config.trace.v3 import service_pb2 -from envoy.config.trace.v3 import skywalking_pb2 -from envoy.config.trace.v2 import http_tracer_pb2 -from envoy.config.trace.v2 import zipkin_pb2 -from envoy.config.trace.v2 import trace_pb2 -from envoy.config.trace.v2 import lightstep_pb2 -from envoy.config.trace.v2 import datadog_pb2 -from envoy.config.trace.v2 import opencensus_pb2 -from envoy.config.trace.v2 import dynamic_ot_pb2 -from envoy.config.trace.v2 import service_pb2 -from envoy.config.cluster.dynamic_forward_proxy.v2alpha import cluster_pb2 -from envoy.config.cluster.redis import redis_cluster_pb2 -from envoy.config.cluster.v3 import filter_pb2 -from envoy.config.cluster.v3 import cluster_pb2 -from envoy.config.cluster.v3 import circuit_breaker_pb2 -from envoy.config.cluster.v3 import outlier_detection_pb2 -from envoy.config.cluster.aggregate.v2alpha import cluster_pb2 -from envoy.config.rbac.v3 import rbac_pb2 -from envoy.config.rbac.v2 import rbac_pb2 -from envoy.config.endpoint.v3 import endpoint_components_pb2 -from envoy.config.endpoint.v3 import load_report_pb2 -from envoy.config.endpoint.v3 import endpoint_pb2 -from envoy.config.resource_monitor.fixed_heap.v2alpha import fixed_heap_pb2 -from envoy.config.resource_monitor.injected_resource.v2alpha import ( - injected_resource_pb2, -) -from envoy.config.grpc_credential.v2alpha import aws_iam_pb2 -from envoy.config.grpc_credential.v2alpha import file_based_metadata_pb2 -from envoy.config.grpc_credential.v3 import aws_iam_pb2 -from envoy.config.grpc_credential.v3 import file_based_metadata_pb2 -from envoy.config.accesslog.v3 import accesslog_pb2 -from envoy.config.accesslog.v2 import als_pb2 -from envoy.config.accesslog.v2 import file_pb2 -from envoy.config.tap.v3 import common_pb2 -from envoy.config.route.v3 import route_components_pb2 -from envoy.config.route.v3 import scoped_route_pb2 -from envoy.config.route.v3 import route_pb2 -from envoy.config.filter.listener.original_dst.v2 import original_dst_pb2 -from envoy.config.filter.listener.tls_inspector.v2 import tls_inspector_pb2 -from envoy.config.filter.listener.proxy_protocol.v2 import proxy_protocol_pb2 -from envoy.config.filter.listener.http_inspector.v2 import http_inspector_pb2 -from envoy.config.filter.listener.original_src.v2alpha1 import original_src_pb2 -from envoy.config.filter.network.mongo_proxy.v2 import mongo_proxy_pb2 -from envoy.config.filter.network.rate_limit.v2 import rate_limit_pb2 -from envoy.config.filter.network.ext_authz.v2 import ext_authz_pb2 -from envoy.config.filter.network.client_ssl_auth.v2 import client_ssl_auth_pb2 -from envoy.config.filter.network.thrift_proxy.v2alpha1 import thrift_proxy_pb2 -from envoy.config.filter.network.thrift_proxy.v2alpha1 import route_pb2 -from envoy.config.filter.network.kafka_broker.v2alpha1 import kafka_broker_pb2 -from envoy.config.filter.network.zookeeper_proxy.v1alpha1 import ( - zookeeper_proxy_pb2, -) -from envoy.config.filter.network.dubbo_proxy.v2alpha1 import route_pb2 -from envoy.config.filter.network.dubbo_proxy.v2alpha1 import dubbo_proxy_pb2 -from envoy.config.filter.network.rbac.v2 import rbac_pb2 -from envoy.config.filter.network.tcp_proxy.v2 import tcp_proxy_pb2 -from envoy.config.filter.network.echo.v2 import echo_pb2 -from envoy.config.filter.network.direct_response.v2 import config_pb2 -from envoy.config.filter.network.local_rate_limit.v2alpha import ( - local_rate_limit_pb2, -) -from envoy.config.filter.network.sni_cluster.v2 import sni_cluster_pb2 -from envoy.config.filter.network.redis_proxy.v2 import redis_proxy_pb2 -from envoy.config.filter.network.http_connection_manager.v2 import ( - http_connection_manager_pb2, -) -from envoy.config.filter.network.mysql_proxy.v1alpha1 import mysql_proxy_pb2 -from envoy.config.filter.dubbo.router.v2alpha1 import router_pb2 -from envoy.config.filter.http.dynamic_forward_proxy.v2alpha import ( - dynamic_forward_proxy_pb2, -) -from envoy.config.filter.http.gzip.v2 import gzip_pb2 -from envoy.config.filter.http.grpc_http1_reverse_bridge.v2alpha1 import ( - config_pb2, -) -from envoy.config.filter.http.buffer.v2 import buffer_pb2 -from envoy.config.filter.http.cors.v2 import cors_pb2 -from envoy.config.filter.http.rate_limit.v2 import rate_limit_pb2 -from envoy.config.filter.http.health_check.v2 import health_check_pb2 -from envoy.config.filter.http.ext_authz.v2 import ext_authz_pb2 -from envoy.config.filter.http.compressor.v2 import compressor_pb2 -from envoy.config.filter.http.cache.v2alpha import cache_pb2 -from envoy.config.filter.http.adaptive_concurrency.v2alpha import ( - adaptive_concurrency_pb2, -) -from envoy.config.filter.http.on_demand.v2 import on_demand_pb2 -from envoy.config.filter.http.header_to_metadata.v2 import ( - header_to_metadata_pb2, -) -from envoy.config.filter.http.aws_request_signing.v2alpha import ( - aws_request_signing_pb2, -) -from envoy.config.filter.http.rbac.v2 import rbac_pb2 -from envoy.config.filter.http.transcoder.v2 import transcoder_pb2 -from envoy.config.filter.http.dynamo.v2 import dynamo_pb2 -from envoy.config.filter.http.csrf.v2 import csrf_pb2 -from envoy.config.filter.http.aws_lambda.v2alpha import aws_lambda_pb2 -from envoy.config.filter.http.tap.v2alpha import tap_pb2 -from envoy.config.filter.http.grpc_http1_bridge.v2 import config_pb2 -from envoy.config.filter.http.lua.v2 import lua_pb2 -from envoy.config.filter.http.ip_tagging.v2 import ip_tagging_pb2 -from envoy.config.filter.http.grpc_stats.v2alpha import config_pb2 -from envoy.config.filter.http.router.v2 import router_pb2 -from envoy.config.filter.http.fault.v2 import fault_pb2 -from envoy.config.filter.http.jwt_authn.v2alpha import config_pb2 -from envoy.config.filter.http.grpc_web.v2 import grpc_web_pb2 -from envoy.config.filter.http.squash.v2 import squash_pb2 -from envoy.config.filter.http.original_src.v2alpha1 import original_src_pb2 -from envoy.config.filter.accesslog.v2 import accesslog_pb2 -from envoy.config.filter.thrift.rate_limit.v2alpha1 import rate_limit_pb2 -from envoy.config.filter.thrift.router.v2alpha1 import router_pb2 -from envoy.config.filter.udp.udp_proxy.v2alpha import udp_proxy_pb2 -from envoy.config.filter.fault.v2 import fault_pb2 -from envoy.config.bootstrap.v3 import bootstrap_pb2 -from envoy.config.bootstrap.v2 import bootstrap_pb2 -from envoy.config.overload.v2alpha import overload_pb2 -from envoy.config.overload.v3 import overload_pb2 -from envoy.extensions.internal_redirect.previous_routes.v3 import ( - previous_routes_config_pb2, -) -from envoy.extensions.internal_redirect.allow_listed_routes.v3 import ( - allow_listed_routes_config_pb2, -) -from envoy.extensions.internal_redirect.safe_cross_scheme.v3 import ( - safe_cross_scheme_config_pb2, -) -from envoy.extensions.rate_limit_descriptors.expr.v3 import expr_pb2 -from envoy.extensions.udp_packet_writer.v3 import ( - udp_gso_batch_writer_factory_pb2, -) -from envoy.extensions.udp_packet_writer.v3 import udp_default_writer_factory_pb2 -from envoy.extensions.transport_sockets.s2a.v3 import s2a_pb2 -from envoy.extensions.transport_sockets.alts.v3 import alts_pb2 -from envoy.extensions.transport_sockets.raw_buffer.v3 import raw_buffer_pb2 -from envoy.extensions.transport_sockets.quic.v3 import quic_transport_pb2 -from envoy.extensions.transport_sockets.tls.v3 import cert_pb2 -from envoy.extensions.transport_sockets.tls.v3 import common_pb2 -from envoy.extensions.transport_sockets.tls.v3 import ( - tls_spiffe_validator_config_pb2, -) -from envoy.extensions.transport_sockets.tls.v3 import tls_pb2 -from envoy.extensions.transport_sockets.tls.v3 import secret_pb2 -from envoy.extensions.transport_sockets.http_11_proxy.v3 import ( - upstream_http_11_connect_pb2, -) -from envoy.extensions.transport_sockets.starttls.v3 import starttls_pb2 -from envoy.extensions.transport_sockets.proxy_protocol.v3 import ( - upstream_proxy_protocol_pb2, -) -from envoy.extensions.transport_sockets.tap.v3 import tap_pb2 -from envoy.extensions.transport_sockets.internal_upstream.v3 import ( - internal_upstream_pb2, -) -from envoy.extensions.transport_sockets.tcp_stats.v3 import tcp_stats_pb2 -from envoy.extensions.config.validators.minimum_clusters.v3 import ( - minimum_clusters_pb2, -) -from envoy.extensions.stat_sinks.open_telemetry.v3 import open_telemetry_pb2 -from envoy.extensions.stat_sinks.graphite_statsd.v3 import graphite_statsd_pb2 -from envoy.extensions.stat_sinks.wasm.v3 import wasm_pb2 -from envoy.extensions.retry.host.previous_hosts.v3 import previous_hosts_pb2 -from envoy.extensions.retry.host.omit_host_metadata.v3 import ( - omit_host_metadata_config_pb2, -) -from envoy.extensions.retry.host.omit_canary_hosts.v3 import ( - omit_canary_hosts_pb2, -) -from envoy.extensions.retry.priority.previous_priorities.v3 import ( - previous_priorities_config_pb2, -) -from envoy.extensions.common.dynamic_forward_proxy.v3 import dns_cache_pb2 -from envoy.extensions.common.matching.v3 import extension_matcher_pb2 -from envoy.extensions.common.ratelimit.v3 import ratelimit_pb2 -from envoy.extensions.common.tap.v3 import common_pb2 -from envoy.extensions.common.async_files.v3 import async_file_manager_pb2 -from envoy.extensions.network.dns_resolver.cares.v3 import ( - cares_dns_resolver_pb2, -) -from envoy.extensions.network.dns_resolver.getaddrinfo.v3 import ( - getaddrinfo_dns_resolver_pb2, -) -from envoy.extensions.network.dns_resolver.apple.v3 import ( - apple_dns_resolver_pb2, -) -from envoy.extensions.network.socket_interface.v3 import ( - default_socket_interface_pb2, -) -from envoy.extensions.matching.common_inputs.network.v3 import ( - network_inputs_pb2, -) -from envoy.extensions.matching.common_inputs.environment_variable.v3 import ( - input_pb2, -) -from envoy.extensions.matching.common_inputs.ssl.v3 import ssl_inputs_pb2 -from envoy.extensions.matching.input_matchers.consistent_hashing.v3 import ( - consistent_hashing_pb2, -) -from envoy.extensions.matching.input_matchers.ip.v3 import ip_pb2 -from envoy.extensions.matching.input_matchers.runtime_fraction.v3 import ( - runtime_fraction_pb2, -) -from envoy.extensions.load_balancing_policies.common.v3 import common_pb2 -from envoy.extensions.load_balancing_policies.random.v3 import random_pb2 -from envoy.extensions.load_balancing_policies.subset.v3 import subset_pb2 -from envoy.extensions.load_balancing_policies.pick_first.v3 import ( - pick_first_pb2, -) -from envoy.extensions.load_balancing_policies.ring_hash.v3 import ring_hash_pb2 -from envoy.extensions.load_balancing_policies.cluster_provided.v3 import ( - cluster_provided_pb2, -) -from envoy.extensions.load_balancing_policies.maglev.v3 import maglev_pb2 -from envoy.extensions.load_balancing_policies.least_request.v3 import ( - least_request_pb2, -) -from envoy.extensions.load_balancing_policies.round_robin.v3 import ( - round_robin_pb2, -) -from envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3 import ( - client_side_weighted_round_robin_pb2, -) -from envoy.extensions.load_balancing_policies.wrr_locality.v3 import ( - wrr_locality_pb2, -) -from envoy.extensions.health_check.event_sinks.file.v3 import file_pb2 -from envoy.extensions.early_data.v3 import default_early_data_policy_pb2 -from envoy.extensions.watchdog.profile_action.v3 import profile_action_pb2 -from envoy.extensions.http.custom_response.local_response_policy.v3 import ( - local_response_policy_pb2, -) -from envoy.extensions.http.custom_response.redirect_policy.v3 import ( - redirect_policy_pb2, -) -from envoy.extensions.http.stateful_session.cookie.v3 import cookie_pb2 -from envoy.extensions.http.stateful_session.header.v3 import header_pb2 -from envoy.extensions.http.early_header_mutation.header_mutation.v3 import ( - header_mutation_pb2, -) -from envoy.extensions.http.header_formatters.preserve_case.v3 import ( - preserve_case_pb2, -) -from envoy.extensions.http.original_ip_detection.custom_header.v3 import ( - custom_header_pb2, -) -from envoy.extensions.http.original_ip_detection.xff.v3 import xff_pb2 -from envoy.extensions.http.cache.simple_http_cache.v3 import config_pb2 -from envoy.extensions.http.cache.file_system_http_cache.v3 import ( - file_system_http_cache_pb2, -) -from envoy.extensions.http.header_validators.envoy_default.v3 import ( - header_validator_pb2, -) -from envoy.extensions.request_id.uuid.v3 import uuid_pb2 -from envoy.extensions.formatter.req_without_query.v3 import ( - req_without_query_pb2, -) -from envoy.extensions.formatter.metadata.v3 import metadata_pb2 -from envoy.extensions.formatter.cel.v3 import cel_pb2 -from envoy.extensions.filters.listener.original_dst.v3 import original_dst_pb2 -from envoy.extensions.filters.listener.tls_inspector.v3 import tls_inspector_pb2 -from envoy.extensions.filters.listener.local_ratelimit.v3 import ( - local_ratelimit_pb2, -) -from envoy.extensions.filters.listener.proxy_protocol.v3 import ( - proxy_protocol_pb2, -) -from envoy.extensions.filters.listener.http_inspector.v3 import ( - http_inspector_pb2, -) -from envoy.extensions.filters.listener.original_src.v3 import original_src_pb2 -from envoy.extensions.filters.common.matcher.action.v3 import skip_action_pb2 -from envoy.extensions.filters.common.dependency.v3 import dependency_pb2 -from envoy.extensions.filters.common.fault.v3 import fault_pb2 -from envoy.extensions.filters.network.mongo_proxy.v3 import mongo_proxy_pb2 -from envoy.extensions.filters.network.ext_authz.v3 import ext_authz_pb2 -from envoy.extensions.filters.network.ratelimit.v3 import rate_limit_pb2 -from envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3 import ( - sni_dynamic_forward_proxy_pb2, -) -from envoy.extensions.filters.network.thrift_proxy.v3 import thrift_proxy_pb2 -from envoy.extensions.filters.network.thrift_proxy.v3 import route_pb2 -from envoy.extensions.filters.network.thrift_proxy.filters.ratelimit.v3 import ( - rate_limit_pb2, -) -from envoy.extensions.filters.network.thrift_proxy.filters.header_to_metadata.v3 import ( - header_to_metadata_pb2, -) -from envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3 import ( - payload_to_metadata_pb2, -) -from envoy.extensions.filters.network.thrift_proxy.router.v3 import router_pb2 -from envoy.extensions.filters.network.zookeeper_proxy.v3 import ( - zookeeper_proxy_pb2, -) -from envoy.extensions.filters.network.dubbo_proxy.v3 import route_pb2 -from envoy.extensions.filters.network.dubbo_proxy.v3 import dubbo_proxy_pb2 -from envoy.extensions.filters.network.dubbo_proxy.router.v3 import router_pb2 -from envoy.extensions.filters.network.rbac.v3 import rbac_pb2 -from envoy.extensions.filters.network.local_ratelimit.v3 import ( - local_rate_limit_pb2, -) -from envoy.extensions.filters.network.connection_limit.v3 import ( - connection_limit_pb2, -) -from envoy.extensions.filters.network.tcp_proxy.v3 import tcp_proxy_pb2 -from envoy.extensions.filters.network.echo.v3 import echo_pb2 -from envoy.extensions.filters.network.direct_response.v3 import config_pb2 -from envoy.extensions.filters.network.sni_cluster.v3 import sni_cluster_pb2 -from envoy.extensions.filters.network.redis_proxy.v3 import redis_proxy_pb2 -from envoy.extensions.filters.network.http_connection_manager.v3 import ( - http_connection_manager_pb2, -) -from envoy.extensions.filters.network.wasm.v3 import wasm_pb2 -from envoy.extensions.filters.http.custom_response.v3 import custom_response_pb2 -from envoy.extensions.filters.http.dynamic_forward_proxy.v3 import ( - dynamic_forward_proxy_pb2, -) -from envoy.extensions.filters.http.oauth2.v3 import oauth_pb2 -from envoy.extensions.filters.http.gzip.v3 import gzip_pb2 -from envoy.extensions.filters.http.grpc_http1_reverse_bridge.v3 import ( - config_pb2, -) -from envoy.extensions.filters.http.buffer.v3 import buffer_pb2 -from envoy.extensions.filters.http.cors.v3 import cors_pb2 -from envoy.extensions.filters.http.decompressor.v3 import decompressor_pb2 -from envoy.extensions.filters.http.stateful_session.v3 import ( - stateful_session_pb2, -) -from envoy.extensions.filters.http.health_check.v3 import health_check_pb2 -from envoy.extensions.filters.http.ext_authz.v3 import ext_authz_pb2 -from envoy.extensions.filters.http.ratelimit.v3 import rate_limit_pb2 -from envoy.extensions.filters.http.geoip.v3 import geoip_pb2 -from envoy.extensions.filters.http.compressor.v3 import compressor_pb2 -from envoy.extensions.filters.http.cache.v3 import cache_pb2 -from envoy.extensions.filters.http.adaptive_concurrency.v3 import ( - adaptive_concurrency_pb2, -) -from envoy.extensions.filters.http.kill_request.v3 import kill_request_pb2 -from envoy.extensions.filters.http.admission_control.v3 import ( - admission_control_pb2, -) -from envoy.extensions.filters.http.on_demand.v3 import on_demand_pb2 -from envoy.extensions.filters.http.header_to_metadata.v3 import ( - header_to_metadata_pb2, -) -from envoy.extensions.filters.http.aws_request_signing.v3 import ( - aws_request_signing_pb2, -) -from envoy.extensions.filters.http.rbac.v3 import rbac_pb2 -from envoy.extensions.filters.http.cdn_loop.v3 import cdn_loop_pb2 -from envoy.extensions.filters.http.composite.v3 import composite_pb2 -from envoy.extensions.filters.http.csrf.v3 import csrf_pb2 -from envoy.extensions.filters.http.local_ratelimit.v3 import ( - local_rate_limit_pb2, -) -from envoy.extensions.filters.http.aws_lambda.v3 import aws_lambda_pb2 -from envoy.extensions.filters.http.tap.v3 import tap_pb2 -from envoy.extensions.filters.http.connect_grpc_bridge.v3 import config_pb2 -from envoy.extensions.filters.http.header_mutation.v3 import header_mutation_pb2 -from envoy.extensions.filters.http.ext_proc.v3 import processing_mode_pb2 -from envoy.extensions.filters.http.ext_proc.v3 import ext_proc_pb2 -from envoy.extensions.filters.http.grpc_http1_bridge.v3 import config_pb2 -from envoy.extensions.filters.http.gcp_authn.v3 import gcp_authn_pb2 -from envoy.extensions.filters.http.alternate_protocols_cache.v3 import ( - alternate_protocols_cache_pb2, -) -from envoy.extensions.filters.http.lua.v3 import lua_pb2 -from envoy.extensions.filters.http.ip_tagging.v3 import ip_tagging_pb2 -from envoy.extensions.filters.http.grpc_stats.v3 import config_pb2 -from envoy.extensions.filters.http.set_metadata.v3 import set_metadata_pb2 -from envoy.extensions.filters.http.router.v3 import router_pb2 -from envoy.extensions.filters.http.fault.v3 import fault_pb2 -from envoy.extensions.filters.http.bandwidth_limit.v3 import bandwidth_limit_pb2 -from envoy.extensions.filters.http.file_system_buffer.v3 import ( - file_system_buffer_pb2, -) -from envoy.extensions.filters.http.jwt_authn.v3 import config_pb2 -from envoy.extensions.filters.http.grpc_web.v3 import grpc_web_pb2 -from envoy.extensions.filters.http.grpc_json_transcoder.v3 import transcoder_pb2 -from envoy.extensions.filters.http.wasm.v3 import wasm_pb2 -from envoy.extensions.filters.http.original_src.v3 import original_src_pb2 -from envoy.extensions.filters.http.rate_limit_quota.v3 import ( - rate_limit_quota_pb2, -) -from envoy.extensions.filters.http.upstream_codec.v3 import upstream_codec_pb2 -from envoy.extensions.filters.udp.udp_proxy.v3 import route_pb2 -from envoy.extensions.filters.udp.udp_proxy.v3 import udp_proxy_pb2 -from envoy.extensions.filters.udp.dns_filter.v3 import dns_filter_pb2 -from envoy.extensions.quic.proof_source.v3 import proof_source_pb2 -from envoy.extensions.quic.crypto_stream.v3 import crypto_stream_pb2 -from envoy.extensions.quic.server_preferred_address.v3 import ( - fixed_server_preferred_address_config_pb2, -) -from envoy.extensions.quic.connection_id_generator.v3 import ( - envoy_deterministic_connection_id_generator_pb2, -) -from envoy.extensions.rbac.audit_loggers.stream.v3 import stream_pb2 -from envoy.extensions.rbac.matchers.upstream_ip_port.v3 import ( - upstream_ip_port_matcher_pb2, -) -from envoy.extensions.path.match.uri_template.v3 import uri_template_match_pb2 -from envoy.extensions.path.rewrite.uri_template.v3 import ( - uri_template_rewrite_pb2, -) -from envoy.extensions.upstreams.tcp.v3 import tcp_protocol_options_pb2 -from envoy.extensions.upstreams.tcp.generic.v3 import ( - generic_connection_pool_pb2, -) -from envoy.extensions.upstreams.http.v3 import http_protocol_options_pb2 -from envoy.extensions.upstreams.http.generic.v3 import ( - generic_connection_pool_pb2, -) -from envoy.extensions.upstreams.http.tcp.v3 import tcp_connection_pool_pb2 -from envoy.extensions.upstreams.http.http.v3 import http_connection_pool_pb2 -from envoy.extensions.compression.gzip.decompressor.v3 import gzip_pb2 -from envoy.extensions.compression.gzip.compressor.v3 import gzip_pb2 -from envoy.extensions.compression.brotli.decompressor.v3 import brotli_pb2 -from envoy.extensions.compression.brotli.compressor.v3 import brotli_pb2 -from envoy.extensions.compression.zstd.decompressor.v3 import zstd_pb2 -from envoy.extensions.compression.zstd.compressor.v3 import zstd_pb2 -from envoy.extensions.resource_monitors.downstream_connections.v3 import ( - downstream_connections_pb2, -) -from envoy.extensions.resource_monitors.fixed_heap.v3 import fixed_heap_pb2 -from envoy.extensions.resource_monitors.injected_resource.v3 import ( - injected_resource_pb2, -) -from envoy.extensions.key_value.file_based.v3 import config_pb2 -from envoy.extensions.health_checkers.redis.v3 import redis_pb2 -from envoy.extensions.health_checkers.thrift.v3 import thrift_pb2 -from envoy.extensions.access_loggers.open_telemetry.v3 import logs_service_pb2 -from envoy.extensions.access_loggers.grpc.v3 import als_pb2 -from envoy.extensions.access_loggers.stream.v3 import stream_pb2 -from envoy.extensions.access_loggers.filters.cel.v3 import cel_pb2 -from envoy.extensions.access_loggers.file.v3 import file_pb2 -from envoy.extensions.access_loggers.wasm.v3 import wasm_pb2 -from envoy.extensions.regex_engines.v3 import google_re2_pb2 -from envoy.extensions.clusters.dynamic_forward_proxy.v3 import cluster_pb2 -from envoy.extensions.clusters.redis.v3 import redis_cluster_pb2 -from envoy.extensions.clusters.aggregate.v3 import cluster_pb2 -from envoy.extensions.bootstrap.internal_listener.v3 import ( - internal_listener_pb2, -) -from envoy.extensions.wasm.v3 import wasm_pb2 -from envoy.data.core.v2alpha import health_check_event_pb2 -from envoy.data.core.v3 import health_check_event_pb2 -from envoy.data.cluster.v2alpha import outlier_detection_event_pb2 -from envoy.data.cluster.v3 import outlier_detection_event_pb2 -from envoy.data.dns.v2alpha import dns_table_pb2 -from envoy.data.dns.v3 import dns_table_pb2 -from envoy.data.accesslog.v3 import accesslog_pb2 -from envoy.data.accesslog.v2 import accesslog_pb2 -from envoy.data.tap.v2alpha import common_pb2 -from envoy.data.tap.v2alpha import http_pb2 -from envoy.data.tap.v2alpha import wrapper_pb2 -from envoy.data.tap.v2alpha import transport_pb2 -from envoy.data.tap.v3 import common_pb2 -from envoy.data.tap.v3 import http_pb2 -from envoy.data.tap.v3 import wrapper_pb2 -from envoy.data.tap.v3 import transport_pb2 -from envoy.watchdog.v3 import abort_action_pb2 -from envoy.admin.v2alpha import mutex_stats_pb2 -from envoy.admin.v2alpha import memory_pb2 -from envoy.admin.v2alpha import server_info_pb2 -from envoy.admin.v2alpha import certs_pb2 -from envoy.admin.v2alpha import tap_pb2 -from envoy.admin.v2alpha import metrics_pb2 -from envoy.admin.v2alpha import config_dump_pb2 -from envoy.admin.v2alpha import clusters_pb2 -from envoy.admin.v2alpha import listeners_pb2 -from envoy.admin.v3 import mutex_stats_pb2 -from envoy.admin.v3 import memory_pb2 -from envoy.admin.v3 import server_info_pb2 -from envoy.admin.v3 import certs_pb2 -from envoy.admin.v3 import tap_pb2 -from envoy.admin.v3 import metrics_pb2 -from envoy.admin.v3 import config_dump_pb2 -from envoy.admin.v3 import clusters_pb2 -from envoy.admin.v3 import init_dump_pb2 -from envoy.admin.v3 import listeners_pb2 -from envoy.admin.v3 import config_dump_shared_pb2 -from envoy.service.load_stats.v3 import lrs_pb2 -from envoy.service.load_stats.v3 import lrs_pb2_grpc -from envoy.service.load_stats.v2 import lrs_pb2 -from envoy.service.load_stats.v2 import lrs_pb2_grpc -from envoy.service.listener.v3 import lds_pb2 -from envoy.service.listener.v3 import lds_pb2_grpc -from envoy.service.extension.v3 import config_discovery_pb2 -from envoy.service.extension.v3 import config_discovery_pb2_grpc -from envoy.service.ratelimit.v3 import rls_pb2 -from envoy.service.ratelimit.v3 import rls_pb2_grpc -from envoy.service.ratelimit.v2 import rls_pb2 -from envoy.service.ratelimit.v2 import rls_pb2_grpc -from envoy.service.trace.v3 import trace_service_pb2 -from envoy.service.trace.v3 import trace_service_pb2_grpc -from envoy.service.trace.v2 import trace_service_pb2 -from envoy.service.trace.v2 import trace_service_pb2_grpc -from envoy.service.cluster.v3 import cds_pb2 -from envoy.service.cluster.v3 import cds_pb2_grpc -from envoy.service.endpoint.v3 import leds_pb2 -from envoy.service.endpoint.v3 import leds_pb2_grpc -from envoy.service.endpoint.v3 import eds_pb2 -from envoy.service.endpoint.v3 import eds_pb2_grpc -from envoy.service.auth.v2alpha import external_auth_pb2 -from envoy.service.auth.v2alpha import external_auth_pb2_grpc -from envoy.service.auth.v3 import external_auth_pb2 -from envoy.service.auth.v3 import external_auth_pb2_grpc -from envoy.service.auth.v3 import attribute_context_pb2 -from envoy.service.auth.v3 import attribute_context_pb2_grpc -from envoy.service.auth.v2 import external_auth_pb2 -from envoy.service.auth.v2 import external_auth_pb2_grpc -from envoy.service.auth.v2 import attribute_context_pb2 -from envoy.service.auth.v2 import attribute_context_pb2_grpc -from envoy.service.accesslog.v3 import als_pb2 -from envoy.service.accesslog.v3 import als_pb2_grpc -from envoy.service.accesslog.v2 import als_pb2 -from envoy.service.accesslog.v2 import als_pb2_grpc -from envoy.service.tap.v2alpha import tap_pb2 -from envoy.service.tap.v2alpha import tap_pb2_grpc -from envoy.service.tap.v2alpha import common_pb2 -from envoy.service.tap.v2alpha import common_pb2_grpc -from envoy.service.tap.v3 import tap_pb2 -from envoy.service.tap.v3 import tap_pb2_grpc -from envoy.service.ext_proc.v3 import external_processor_pb2 -from envoy.service.ext_proc.v3 import external_processor_pb2_grpc -from envoy.service.route.v3 import rds_pb2 -from envoy.service.route.v3 import rds_pb2_grpc -from envoy.service.route.v3 import srds_pb2 -from envoy.service.route.v3 import srds_pb2_grpc -from envoy.service.event_reporting.v2alpha import event_reporting_service_pb2 -from envoy.service.event_reporting.v2alpha import ( - event_reporting_service_pb2_grpc, -) -from envoy.service.event_reporting.v3 import event_reporting_service_pb2 -from envoy.service.event_reporting.v3 import event_reporting_service_pb2_grpc -from envoy.service.runtime.v3 import rtds_pb2 -from envoy.service.runtime.v3 import rtds_pb2_grpc -from envoy.service.health.v3 import hds_pb2 -from envoy.service.health.v3 import hds_pb2_grpc -from envoy.service.status.v3 import csds_pb2 -from envoy.service.status.v3 import csds_pb2_grpc -from envoy.service.status.v2 import csds_pb2 -from envoy.service.status.v2 import csds_pb2_grpc -from envoy.service.rate_limit_quota.v3 import rlqs_pb2 -from envoy.service.rate_limit_quota.v3 import rlqs_pb2_grpc -from envoy.service.discovery.v3 import ads_pb2 -from envoy.service.discovery.v3 import ads_pb2_grpc -from envoy.service.discovery.v3 import discovery_pb2 -from envoy.service.discovery.v3 import discovery_pb2_grpc -from envoy.service.discovery.v2 import ads_pb2 -from envoy.service.discovery.v2 import ads_pb2_grpc -from envoy.service.discovery.v2 import sds_pb2 -from envoy.service.discovery.v2 import sds_pb2_grpc -from envoy.service.discovery.v2 import hds_pb2 -from envoy.service.discovery.v2 import hds_pb2_grpc -from envoy.service.discovery.v2 import rtds_pb2 -from envoy.service.discovery.v2 import rtds_pb2_grpc -from envoy.service.secret.v3 import sds_pb2 -from envoy.service.secret.v3 import sds_pb2_grpc -from envoy.type import range_pb2 -from envoy.type import token_bucket_pb2 -from envoy.type import hash_policy_pb2 -from envoy.type import semantic_version_pb2 -from envoy.type import http_status_pb2 -from envoy.type import http_pb2 -from envoy.type import percent_pb2 -from envoy.type.v3 import range_pb2 -from envoy.type.v3 import token_bucket_pb2 -from envoy.type.v3 import ratelimit_strategy_pb2 -from envoy.type.v3 import hash_policy_pb2 -from envoy.type.v3 import ratelimit_unit_pb2 -from envoy.type.v3 import semantic_version_pb2 -from envoy.type.v3 import http_status_pb2 -from envoy.type.v3 import http_pb2 -from envoy.type.v3 import percent_pb2 -from envoy.type.http.v3 import path_transformation_pb2 -from envoy.type.http.v3 import cookie_pb2 -from envoy.type.matcher import struct_pb2 -from envoy.type.matcher import path_pb2 -from envoy.type.matcher import regex_pb2 -from envoy.type.matcher import number_pb2 -from envoy.type.matcher import metadata_pb2 -from envoy.type.matcher import string_pb2 -from envoy.type.matcher import node_pb2 -from envoy.type.matcher import value_pb2 -from envoy.type.matcher.v3 import struct_pb2 -from envoy.type.matcher.v3 import http_inputs_pb2 -from envoy.type.matcher.v3 import path_pb2 -from envoy.type.matcher.v3 import regex_pb2 -from envoy.type.matcher.v3 import status_code_input_pb2 -from envoy.type.matcher.v3 import number_pb2 -from envoy.type.matcher.v3 import metadata_pb2 -from envoy.type.matcher.v3 import string_pb2 -from envoy.type.matcher.v3 import node_pb2 -from envoy.type.matcher.v3 import value_pb2 -from envoy.type.matcher.v3 import filter_state_pb2 -from envoy.type.metadata.v3 import metadata_pb2 -from envoy.type.metadata.v2 import metadata_pb2 -from envoy.type.tracing.v3 import custom_tag_pb2 -from envoy.type.tracing.v2 import custom_tag_pb2 -from envoy.annotations import deprecation_pb2 -from envoy.annotations import resource_pb2 -from envoy.api.v2 import rds_pb2 -from envoy.api.v2 import lds_pb2 -from envoy.api.v2 import scoped_route_pb2 -from envoy.api.v2 import route_pb2 -from envoy.api.v2 import discovery_pb2 -from envoy.api.v2 import cds_pb2 -from envoy.api.v2 import cluster_pb2 -from envoy.api.v2 import eds_pb2 -from envoy.api.v2 import srds_pb2 -from envoy.api.v2 import listener_pb2 -from envoy.api.v2 import endpoint_pb2 -from envoy.api.v2.listener import listener_components_pb2 -from envoy.api.v2.listener import udp_listener_config_pb2 -from envoy.api.v2.listener import quic_config_pb2 -from envoy.api.v2.listener import listener_pb2 -from envoy.api.v2.core import base_pb2 -from envoy.api.v2.core import backoff_pb2 -from envoy.api.v2.core import grpc_service_pb2 -from envoy.api.v2.core import protocol_pb2 -from envoy.api.v2.core import address_pb2 -from envoy.api.v2.core import health_check_pb2 -from envoy.api.v2.core import grpc_method_list_pb2 -from envoy.api.v2.core import socket_option_pb2 -from envoy.api.v2.core import config_source_pb2 -from envoy.api.v2.core import event_service_config_pb2 -from envoy.api.v2.core import http_uri_pb2 -from envoy.api.v2.ratelimit import ratelimit_pb2 -from envoy.api.v2.cluster import filter_pb2 -from envoy.api.v2.cluster import circuit_breaker_pb2 -from envoy.api.v2.cluster import outlier_detection_pb2 -from envoy.api.v2.endpoint import endpoint_components_pb2 -from envoy.api.v2.endpoint import load_report_pb2 -from envoy.api.v2.endpoint import endpoint_pb2 -from envoy.api.v2.auth import cert_pb2 -from envoy.api.v2.auth import common_pb2 -from envoy.api.v2.auth import tls_pb2 -from envoy.api.v2.auth import secret_pb2 -from envoy.api.v2.route import route_components_pb2 -from envoy.api.v2.route import route_pb2 -from xds.core.v3 import cidr_pb2 -from xds.core.v3 import authority_pb2 -from xds.core.v3 import resource_locator_pb2 -from xds.core.v3 import resource_name_pb2 -from xds.core.v3 import context_params_pb2 -from xds.core.v3 import resource_pb2 -from xds.core.v3 import extension_pb2 -from xds.core.v3 import collection_entry_pb2 -from xds.data.orca.v3 import orca_load_report_pb2 -from xds.service.orca.v3 import orca_pb2 -from xds.type.v3 import range_pb2 -from xds.type.v3 import cel_pb2 -from xds.type.v3 import typed_struct_pb2 -from xds.type.matcher.v3 import range_pb2 -from xds.type.matcher.v3 import http_inputs_pb2 -from xds.type.matcher.v3 import domain_pb2 -from xds.type.matcher.v3 import regex_pb2 -from xds.type.matcher.v3 import cel_pb2 -from xds.type.matcher.v3 import matcher_pb2 -from xds.type.matcher.v3 import string_pb2 -from xds.type.matcher.v3 import ip_pb2 -from xds.annotations.v3 import versioning_pb2 -from xds.annotations.v3 import migrate_pb2 -from xds.annotations.v3 import sensitive_pb2 -from xds.annotations.v3 import status_pb2 -from xds.annotations.v3 import security_pb2 -from udpa.data.orca.v1 import orca_load_report_pb2 -from udpa.service.orca.v1 import orca_pb2 -from udpa.type.v1 import typed_struct_pb2 -from udpa.annotations import versioning_pb2 -from udpa.annotations import migrate_pb2 -from udpa.annotations import sensitive_pb2 -from udpa.annotations import status_pb2 -from udpa.annotations import security_pb2 -from google.api import context_pb2 -from google.api import visibility_pb2 -from google.api import config_change_pb2 -from google.api import source_info_pb2 -from google.api import field_behavior_pb2 -from google.api import monitored_resource_pb2 -from google.api import metric_pb2 -from google.api import usage_pb2 -from google.api import backend_pb2 -from google.api import monitoring_pb2 -from google.api import control_pb2 -from google.api import billing_pb2 -from google.api import system_parameter_pb2 -from google.api import auth_pb2 -from google.api import quota_pb2 -from google.api import client_pb2 -from google.api import documentation_pb2 -from google.api import http_pb2 -from google.api import resource_pb2 -from google.api import annotations_pb2 -from google.api import log_pb2 -from google.api import httpbody_pb2 -from google.api import service_pb2 -from google.api import launch_stage_pb2 -from google.api import consumer_pb2 -from google.api import endpoint_pb2 -from google.api import label_pb2 -from google.api import distribution_pb2 -from google.api import logging_pb2 -from google.api import error_reason_pb2 -from google.api.servicecontrol.v1 import log_entry_pb2 -from google.api.servicecontrol.v1 import metric_value_pb2 -from google.api.servicecontrol.v1 import operation_pb2 -from google.api.servicecontrol.v1 import service_controller_pb2 -from google.api.servicecontrol.v1 import http_request_pb2 -from google.api.servicecontrol.v1 import quota_controller_pb2 -from google.api.servicecontrol.v1 import check_error_pb2 -from google.api.servicecontrol.v1 import distribution_pb2 -from google.api.servicemanagement.v1 import resources_pb2 -from google.api.servicemanagement.v1 import servicemanager_pb2 -from google.api.expr.v1beta1 import source_pb2 -from google.api.expr.v1beta1 import eval_pb2 -from google.api.expr.v1beta1 import expr_pb2 -from google.api.expr.v1beta1 import value_pb2 -from google.api.expr.v1beta1 import decl_pb2 -from google.api.expr.v1alpha1 import explain_pb2 -from google.api.expr.v1alpha1 import eval_pb2 -from google.api.expr.v1alpha1 import syntax_pb2 -from google.api.expr.v1alpha1 import checked_pb2 -from google.api.expr.v1alpha1 import conformance_service_pb2 -from google.api.expr.v1alpha1 import value_pb2 -from google.api.serviceusage.v1 import resources_pb2 -from google.api.serviceusage.v1 import serviceusage_pb2 -from google.api.serviceusage.v1beta1 import resources_pb2 -from google.api.serviceusage.v1beta1 import serviceusage_pb2 -from google.rpc import code_pb2 -from google.rpc import error_details_pb2 -from google.rpc import status_pb2 -from google.rpc.context import attribute_context_pb2 -from google.longrunning import operations_pb2 -from google.logging.v2 import logging_metrics_pb2 -from google.logging.v2 import log_entry_pb2 -from google.logging.v2 import logging_config_pb2 -from google.logging.v2 import logging_pb2 -from google.logging.type import http_request_pb2 -from google.logging.type import log_severity_pb2 -from google.type import calendar_period_pb2 -from google.type import datetime_pb2 -from google.type import color_pb2 -from google.type import phone_number_pb2 -from google.type import money_pb2 -from google.type import timeofday_pb2 -from google.type import decimal_pb2 -from google.type import postal_address_pb2 -from google.type import date_pb2 -from google.type import expr_pb2 -from google.type import interval_pb2 -from google.type import localized_text_pb2 -from google.type import dayofweek_pb2 -from google.type import quaternion_pb2 -from google.type import month_pb2 -from google.type import latlng_pb2 -from google.type import fraction_pb2 -from validate import validate_pb2 -from opencensus.proto.metrics.v1 import metrics_pb2 -from opencensus.proto.agent.common.v1 import common_pb2 -from opencensus.proto.agent.metrics.v1 import metrics_service_pb2 -from opencensus.proto.agent.trace.v1 import trace_service_pb2 -from opencensus.proto.trace.v1 import trace_config_pb2 -from opencensus.proto.trace.v1 import trace_pb2 -from opencensus.proto.stats.v1 import stats_pb2 -from opencensus.proto.resource.v1 import resource_pb2 -from opentelemetry.proto.common.v1 import common_pb2 -from opentelemetry.proto.metrics.v1 import metrics_pb2 -from opentelemetry.proto.metrics.experimental import metrics_config_service_pb2 -from opentelemetry.proto.trace.v1 import trace_config_pb2 -from opentelemetry.proto.trace.v1 import trace_pb2 -from opentelemetry.proto.logs.v1 import logs_pb2 -from opentelemetry.proto.collector.metrics.v1 import metrics_service_pb2 -from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 -from opentelemetry.proto.collector.logs.v1 import logs_service_pb2 -from opentelemetry.proto.resource.v1 import resource_pb2 diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py deleted file mode 100644 index 5da8a088908..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/client_app.py +++ /dev/null @@ -1,535 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Provides an interface to xDS Test Client running remotely. -""" -import datetime -import functools -import logging -import time -from typing import Iterable, List, Optional - -import framework.errors -from framework.helpers import retryers -import framework.rpc -from framework.rpc import grpc_channelz -from framework.rpc import grpc_csds -from framework.rpc import grpc_testing - -logger = logging.getLogger(__name__) - -# Type aliases -_timedelta = datetime.timedelta -_LoadBalancerStatsServiceClient = grpc_testing.LoadBalancerStatsServiceClient -_XdsUpdateClientConfigureServiceClient = ( - grpc_testing.XdsUpdateClientConfigureServiceClient -) -_ChannelzServiceClient = grpc_channelz.ChannelzServiceClient -_ChannelzChannel = grpc_channelz.Channel -_ChannelzChannelData = grpc_channelz.ChannelData -_ChannelzChannelState = grpc_channelz.ChannelState -_ChannelzSubchannel = grpc_channelz.Subchannel -_ChannelzSocket = grpc_channelz.Socket -_CsdsClient = grpc_csds.CsdsClient - -# Use in get_load_balancer_stats request to request all metadata. -REQ_LB_STATS_METADATA_ALL = ("*",) - -DEFAULT_TD_XDS_URI = "trafficdirector.googleapis.com:443" - - -class XdsTestClient(framework.rpc.grpc.GrpcApp): - """ - Represents RPC services implemented in Client component of the xds test app. - https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#client - """ - - # A unique string identifying each client replica. Used in logging. - hostname: str - - def __init__( - self, - *, - ip: str, - rpc_port: int, - server_target: str, - hostname: str, - rpc_host: Optional[str] = None, - maintenance_port: Optional[int] = None, - ): - super().__init__(rpc_host=(rpc_host or ip)) - self.ip = ip - self.rpc_port = rpc_port - self.server_target = server_target - self.maintenance_port = maintenance_port or rpc_port - self.hostname = hostname - - @property - @functools.lru_cache(None) - def load_balancer_stats(self) -> _LoadBalancerStatsServiceClient: - return _LoadBalancerStatsServiceClient( - self._make_channel(self.rpc_port), - log_target=f"{self.hostname}:{self.rpc_port}", - ) - - @property - @functools.lru_cache(None) - def update_config(self): - return _XdsUpdateClientConfigureServiceClient( - self._make_channel(self.rpc_port), - log_target=f"{self.hostname}:{self.rpc_port}", - ) - - @property - @functools.lru_cache(None) - def channelz(self) -> _ChannelzServiceClient: - return _ChannelzServiceClient( - self._make_channel(self.maintenance_port), - log_target=f"{self.hostname}:{self.maintenance_port}", - ) - - @property - @functools.lru_cache(None) - def csds(self) -> _CsdsClient: - return _CsdsClient( - self._make_channel(self.maintenance_port), - log_target=f"{self.hostname}:{self.maintenance_port}", - ) - - def get_load_balancer_stats( - self, - *, - num_rpcs: int, - metadata_keys: Optional[tuple[str, ...]] = None, - timeout_sec: Optional[int] = None, - ) -> grpc_testing.LoadBalancerStatsResponse: - """ - Shortcut to LoadBalancerStatsServiceClient.get_client_stats() - """ - return self.load_balancer_stats.get_client_stats( - num_rpcs=num_rpcs, - timeout_sec=timeout_sec, - metadata_keys=metadata_keys, - ) - - def get_load_balancer_accumulated_stats( - self, - *, - timeout_sec: Optional[int] = None, - ) -> grpc_testing.LoadBalancerAccumulatedStatsResponse: - """Shortcut to LoadBalancerStatsServiceClient.get_client_accumulated_stats()""" - return self.load_balancer_stats.get_client_accumulated_stats( - timeout_sec=timeout_sec - ) - - def wait_for_server_channel_ready( - self, - *, - timeout: Optional[_timedelta] = None, - rpc_deadline: Optional[_timedelta] = None, - ) -> _ChannelzChannel: - """Wait for the channel to the server to transition to READY. - - Raises: - GrpcApp.NotFound: If the channel never transitioned to READY. - """ - try: - return self.wait_for_server_channel_state( - _ChannelzChannelState.READY, - timeout=timeout, - rpc_deadline=rpc_deadline, - ) - except retryers.RetryError as retry_err: - if isinstance(retry_err.exception(), self.ChannelNotFound): - retry_err.add_note( - framework.errors.FrameworkError.note_blanket_error( - "The client couldn't connect to the server." - ) - ) - raise - - def wait_for_active_xds_channel( - self, - *, - xds_server_uri: Optional[str] = None, - timeout: Optional[_timedelta] = None, - rpc_deadline: Optional[_timedelta] = None, - ) -> _ChannelzChannel: - """Wait until the xds channel is active or timeout. - - Raises: - GrpcApp.NotFound: If the channel to xds never transitioned to active. - """ - try: - return self.wait_for_xds_channel_active( - xds_server_uri=xds_server_uri, - timeout=timeout, - rpc_deadline=rpc_deadline, - ) - except retryers.RetryError as retry_err: - if isinstance(retry_err.exception(), self.ChannelNotFound): - retry_err.add_note( - framework.errors.FrameworkError.note_blanket_error( - "The client couldn't connect to the xDS control plane." - ) - ) - raise - - def get_active_server_channel_socket(self) -> _ChannelzSocket: - channel = self.find_server_channel_with_state( - _ChannelzChannelState.READY - ) - # Get the first subchannel of the active channel to the server. - logger.debug( - ( - "[%s] Retrieving client -> server socket, " - "channel_id: %s, subchannel: %s" - ), - self.hostname, - channel.ref.channel_id, - channel.subchannel_ref[0].name, - ) - subchannel, *subchannels = list( - self.channelz.list_channel_subchannels(channel) - ) - if subchannels: - logger.warning( - "[%s] Unexpected subchannels: %r", self.hostname, subchannels - ) - # Get the first socket of the subchannel - socket, *sockets = list( - self.channelz.list_subchannels_sockets(subchannel) - ) - if sockets: - logger.warning( - "[%s] Unexpected sockets: %r", self.hostname, subchannels - ) - logger.debug( - "[%s] Found client -> server socket: %s", - self.hostname, - socket.ref.name, - ) - return socket - - def wait_for_server_channel_state( - self, - state: _ChannelzChannelState, - *, - timeout: Optional[_timedelta] = None, - rpc_deadline: Optional[_timedelta] = None, - ) -> _ChannelzChannel: - # When polling for a state, prefer smaller wait times to avoid - # exhausting all allowed time on a single long RPC. - if rpc_deadline is None: - rpc_deadline = _timedelta(seconds=30) - - # Fine-tuned to wait for the channel to the server. - retryer = retryers.exponential_retryer_with_timeout( - wait_min=_timedelta(seconds=10), - wait_max=_timedelta(seconds=25), - timeout=_timedelta(minutes=5) if timeout is None else timeout, - ) - - logger.info( - "[%s] Waiting to report a %s channel to %s", - self.hostname, - _ChannelzChannelState.Name(state), - self.server_target, - ) - channel = retryer( - self.find_server_channel_with_state, - state, - rpc_deadline=rpc_deadline, - ) - logger.info( - "[%s] Channel to %s transitioned to state %s: %s", - self.hostname, - self.server_target, - _ChannelzChannelState.Name(state), - _ChannelzServiceClient.channel_repr(channel), - ) - return channel - - def wait_for_xds_channel_active( - self, - *, - xds_server_uri: Optional[str] = None, - timeout: Optional[_timedelta] = None, - rpc_deadline: Optional[_timedelta] = None, - ) -> _ChannelzChannel: - if not xds_server_uri: - xds_server_uri = DEFAULT_TD_XDS_URI - # When polling for a state, prefer smaller wait times to avoid - # exhausting all allowed time on a single long RPC. - if rpc_deadline is None: - rpc_deadline = _timedelta(seconds=30) - - retryer = retryers.exponential_retryer_with_timeout( - wait_min=_timedelta(seconds=10), - wait_max=_timedelta(seconds=25), - timeout=_timedelta(minutes=5) if timeout is None else timeout, - ) - - logger.info( - "[%s] ADS: Waiting for active calls to xDS control plane to %s", - self.hostname, - xds_server_uri, - ) - channel = retryer( - self.find_active_xds_channel, - xds_server_uri=xds_server_uri, - rpc_deadline=rpc_deadline, - ) - logger.info( - "[%s] ADS: Detected active calls to xDS control plane %s", - self.hostname, - xds_server_uri, - ) - return channel - - def find_active_xds_channel( - self, - xds_server_uri: str, - *, - rpc_deadline: Optional[_timedelta] = None, - ) -> _ChannelzChannel: - rpc_params = {} - if rpc_deadline is not None: - rpc_params["deadline_sec"] = rpc_deadline.total_seconds() - - for channel in self.find_channels(xds_server_uri, **rpc_params): - logger.info( - "[%s] xDS control plane channel: %s", - self.hostname, - _ChannelzServiceClient.channel_repr(channel), - ) - - try: - channel_upd = self.check_channel_in_flight_calls( - channel, **rpc_params - ) - logger.info( - "[%s] Detected active calls to xDS control plane %s," - " channel: %s", - self.hostname, - xds_server_uri, - _ChannelzServiceClient.channel_repr(channel_upd), - ) - return channel_upd - except self.NotFound: - # Continue checking other channels to the same target on - # not found. - continue - except framework.rpc.grpc.RpcError as err: - # Logged at 'info' and not at 'warning' because this method is - # expected to be called in a retryer. If this error eventually - # causes the retryer to fail, it will be logged fully at 'error' - logger.info( - "[%s] Unexpected error while checking xDS control plane" - " channel %s: %r", - self.hostname, - _ChannelzServiceClient.channel_repr(channel), - err, - ) - raise - - raise self.ChannelNotActive( - f"[{self.hostname}] Client has no" - f" active channel with xDS control plane {xds_server_uri}", - src=self.hostname, - dst=xds_server_uri, - ) - - def find_server_channel_with_state( - self, - expected_state: _ChannelzChannelState, - *, - rpc_deadline: Optional[_timedelta] = None, - check_subchannel=True, - ) -> _ChannelzChannel: - rpc_params = {} - if rpc_deadline is not None: - rpc_params["deadline_sec"] = rpc_deadline.total_seconds() - - expected_state_name: str = _ChannelzChannelState.Name(expected_state) - target: str = self.server_target - - for channel in self.find_channels(target, **rpc_params): - channel_state: _ChannelzChannelState = channel.data.state.state - logger.info( - "[%s] Server channel: %s", - self.hostname, - _ChannelzServiceClient.channel_repr(channel), - ) - if channel_state is expected_state: - if check_subchannel: - # When requested, check if the channel has at least - # one subchannel in the requested state. - try: - subchannel = self.find_subchannel_with_state( - channel, expected_state, **rpc_params - ) - logger.info( - "[%s] Found subchannel in state %s: %s", - self.hostname, - expected_state_name, - _ChannelzServiceClient.subchannel_repr(subchannel), - ) - except self.NotFound as e: - # Otherwise, keep searching. - logger.info(e.message) - continue - return channel - - raise self.ChannelNotFound( - f"[{self.hostname}] Client has no" - f" {expected_state_name} channel with server {target}", - src=self.hostname, - dst=target, - expected_state=expected_state, - ) - - def find_channels( - self, - target: str, - **rpc_params, - ) -> Iterable[_ChannelzChannel]: - return self.channelz.find_channels_for_target(target, **rpc_params) - - def find_subchannel_with_state( - self, channel: _ChannelzChannel, state: _ChannelzChannelState, **kwargs - ) -> _ChannelzSubchannel: - subchannels = self.channelz.list_channel_subchannels(channel, **kwargs) - for subchannel in subchannels: - if subchannel.data.state.state is state: - return subchannel - - raise self.NotFound( - f"[{self.hostname}] Not found " - f"a {_ChannelzChannelState.Name(state)} subchannel " - f"for channel_id {channel.ref.channel_id}" - ) - - def find_subchannels_with_state( - self, state: _ChannelzChannelState, **kwargs - ) -> List[_ChannelzSubchannel]: - subchannels = [] - for channel in self.channelz.find_channels_for_target( - self.server_target, **kwargs - ): - for subchannel in self.channelz.list_channel_subchannels( - channel, **kwargs - ): - if subchannel.data.state.state is state: - subchannels.append(subchannel) - return subchannels - - def check_channel_in_flight_calls( - self, - channel: _ChannelzChannel, - *, - wait_between_checks: Optional[_timedelta] = None, - **rpc_params, - ) -> Optional[_ChannelzChannel]: - """Checks if the channel has calls that started, but didn't complete. - - We consider the channel is active if channel is in READY state and - calls_started is greater than calls_failed. - - This method address race where a call to the xDS control plane server - has just started and a channelz request comes in before the call has - had a chance to fail. - - With channels to the xDS control plane, the channel can be READY but the - calls could be failing to initialize, f.e. due to a failure to fetch - OAUTH2 token. To increase the confidence that we have a valid channel - with working OAUTH2 tokens, we check whether the channel is in a READY - state with active calls twice with an interval of 2 seconds between the - two attempts. If the OAUTH2 token is not valid, the call would fail and - be caught in either the first attempt, or the second attempt. It is - possible that between the two attempts, a call fails and a new call is - started, so we also test for equality between the started calls of the - two channelz results. - - There still exists a possibility that a call fails on fetching OAUTH2 - token after 2 seconds (maybe because there is a slowdown in the - system.) If such a case is observed, consider increasing the interval - from 2 seconds to 5 seconds. - - Returns updated channel on success, or None on failure. - """ - if not self.calc_calls_in_flight(channel): - return None - - if not wait_between_checks: - wait_between_checks = _timedelta(seconds=2) - - # Load the channel second time after the timeout. - time.sleep(wait_between_checks.total_seconds()) - channel_upd: _ChannelzChannel = self.channelz.get_channel( - channel.ref.channel_id, **rpc_params - ) - if ( - not self.calc_calls_in_flight(channel_upd) - or channel.data.calls_started != channel_upd.data.calls_started - ): - return None - return channel_upd - - @classmethod - def calc_calls_in_flight(cls, channel: _ChannelzChannel) -> int: - cdata: _ChannelzChannelData = channel.data - if cdata.state.state is not _ChannelzChannelState.READY: - return 0 - - return cdata.calls_started - cdata.calls_succeeded - cdata.calls_failed - - class ChannelNotFound(framework.rpc.grpc.GrpcApp.NotFound): - """Channel with expected status not found""" - - src: str - dst: str - expected_state: object - - def __init__( - self, - message: str, - *, - src: str, - dst: str, - expected_state: _ChannelzChannelState, - **kwargs, - ): - self.src = src - self.dst = dst - self.expected_state = expected_state - super().__init__(message, src, dst, expected_state, **kwargs) - - class ChannelNotActive(framework.rpc.grpc.GrpcApp.NotFound): - """No active channel was found""" - - src: str - dst: str - - def __init__( - self, - message: str, - *, - src: str, - dst: str, - **kwargs, - ): - self.src = src - self.dst = dst - super().__init__(message, src, dst, **kwargs) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/__init__.py deleted file mode 100644 index a87ddec1edd..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/base_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/base_runner.py deleted file mode 100644 index 096712ccab7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/base_runner.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Common functionality for running xDS Test Client and Server remotely. -""" -from abc import ABCMeta -from abc import abstractmethod -import functools -import pathlib -import threading -from typing import Dict, Optional -import urllib.parse - -from absl import flags - -from framework import xds_flags -from framework.helpers import logs - -flags.adopt_module_key_flags(logs) -_LOGS_SUBDIR = "test_app_logs" - - -class RunnerError(Exception): - """Error running xDS Test App running remotely.""" - - -class BaseRunner(metaclass=ABCMeta): - _logs_subdir: Optional[pathlib.Path] = None - _log_stop_event: Optional[threading.Event] = None - - def __init__(self): - if xds_flags.COLLECT_APP_LOGS.value: - self._logs_subdir = logs.log_dir_mkdir(_LOGS_SUBDIR) - self._log_stop_event = threading.Event() - - @property - @functools.lru_cache(None) - def should_collect_logs(self) -> bool: - return self._logs_subdir is not None - - @property - @functools.lru_cache(None) - def logs_subdir(self) -> pathlib.Path: - if not self.should_collect_logs: - raise FileNotFoundError("Log collection is not enabled.") - return self._logs_subdir - - @property - def log_stop_event(self) -> threading.Event: - if not self.should_collect_logs: - raise ValueError("Log collection is not enabled.") - return self._log_stop_event - - def maybe_stop_logging(self): - if self.should_collect_logs and not self.log_stop_event.is_set(): - self.log_stop_event.set() - - @abstractmethod - def run(self, **kwargs): - pass - - @abstractmethod - def cleanup(self, *, force=False): - pass - - @classmethod - def _logs_explorer_link_from_params( - cls, - *, - gcp_ui_url: str, - gcp_project: str, - query: Dict[str, str], - request: Optional[Dict[str, str]] = None, - ) -> str: - req_merged = {"query": cls._logs_explorer_query(query)} - if request is not None: - req_merged.update(request) - - req = cls._logs_explorer_request(req_merged) - return f"https://{gcp_ui_url}/logs/query;{req}?project={gcp_project}" - - @classmethod - def _logs_explorer_query(cls, query: Dict[str, str]) -> str: - return "\n".join(f'{k}="{v}"' for k, v in query.items()) - - @classmethod - def _logs_explorer_request(cls, req: Dict[str, str]) -> str: - return ";".join( - f"{k}={cls._logs_explorer_quote(v)}" for k, v in req.items() - ) - - @classmethod - def _logs_explorer_quote(cls, value: str) -> str: - return urllib.parse.quote_plus(value, safe=":") diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/__init__.py deleted file mode 100644 index a87ddec1edd..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/gamma_server_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/gamma_server_runner.py deleted file mode 100644 index 745c6b8c018..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/gamma_server_runner.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Run xDS Test Client on Kubernetes using Gamma -""" -import datetime -import logging -from typing import List, Optional - -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.test_app.runners.k8s import k8s_xds_server_runner -from framework.test_app.server_app import XdsTestServer - -logger = logging.getLogger(__name__) - - -KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - - -class GammaServerRunner(KubernetesServerRunner): - # Mutable state. - route: Optional[k8s.GammaHttpRoute] = None - frontend_service: Optional[k8s.V1Service] = None - sa_filter: Optional[k8s.GcpSessionAffinityFilter] = None - sa_policy: Optional[k8s.GcpSessionAffinityPolicy] = None - be_policy: Optional[k8s.GcpBackendPolicy] = None - termination_grace_period_seconds: Optional[int] = None - pre_stop_hook: bool = False - - route_name: str - frontend_service_name: str - - def __init__( - self, - k8s_namespace: k8s.KubernetesNamespace, - frontend_service_name: str, - *, - deployment_name: str, - image_name: str, - td_bootstrap_image: str, - network: str = "default", - xds_server_uri: Optional[str] = None, - gcp_api_manager: gcp.api.GcpApiManager, - gcp_project: str, - gcp_service_account: str, - service_account_name: Optional[str] = None, - service_name: Optional[str] = None, - route_name: Optional[str] = None, - neg_name: Optional[str] = None, - deployment_template: str = "server.deployment.yaml", - service_account_template: str = "service-account.yaml", - service_template: str = "gamma/service.yaml", - reuse_service: bool = False, - reuse_namespace: bool = False, - namespace_template: Optional[str] = None, - debug_use_port_forwarding: bool = False, - enable_workload_identity: bool = True, - safilter_name: str = "ssa-filter", - sapolicy_name: str = "ssa-policy", - bepolicy_name: str = "backend-policy", - termination_grace_period_seconds: int = 0, - pre_stop_hook: bool = False, - ): - # pylint: disable=too-many-locals - super().__init__( - k8s_namespace, - deployment_name=deployment_name, - image_name=image_name, - td_bootstrap_image=td_bootstrap_image, - network=network, - xds_server_uri=xds_server_uri, - gcp_api_manager=gcp_api_manager, - gcp_project=gcp_project, - gcp_service_account=gcp_service_account, - service_account_name=service_account_name, - service_name=service_name, - neg_name=neg_name, - deployment_template=deployment_template, - service_account_template=service_account_template, - service_template=service_template, - reuse_service=reuse_service, - reuse_namespace=reuse_namespace, - namespace_template=namespace_template, - debug_use_port_forwarding=debug_use_port_forwarding, - enable_workload_identity=enable_workload_identity, - ) - - self.frontend_service_name = frontend_service_name - self.route_name = route_name or f"route-{deployment_name}" - self.safilter_name = safilter_name - self.sapolicy_name = sapolicy_name - self.bepolicy_name = bepolicy_name - self.termination_grace_period_seconds = termination_grace_period_seconds - self.pre_stop_hook = pre_stop_hook - - def run( # pylint: disable=arguments-differ - self, - *, - test_port: int = KubernetesServerRunner.DEFAULT_TEST_PORT, - maintenance_port: Optional[int] = None, - secure_mode: bool = False, - replica_count: int = 1, - log_to_stdout: bool = False, - bootstrap_version: Optional[str] = None, - route_template: str = "gamma/route_http.yaml", - enable_csm_observability: bool = False, - ) -> List[XdsTestServer]: - if not maintenance_port: - maintenance_port = self._get_default_maintenance_port(secure_mode) - - logger.info( - ( - 'Deploying GAMMA xDS test server "%s" to k8s namespace %s:' - " test_port=%s maintenance_port=%s secure_mode=%s" - " replica_count=%s" - ), - self.deployment_name, - self.k8s_namespace.name, - test_port, - maintenance_port, - False, - replica_count, - ) - # super(k8s_base_runner.KubernetesBaseRunner, self).run() - - if self.reuse_namespace: - self.namespace = self._reuse_namespace() - if not self.namespace: - self.namespace = self._create_namespace( - self.namespace_template, namespace_name=self.k8s_namespace.name - ) - - # Reuse existing if requested, create a new deployment when missing. - # Useful for debugging to avoid NEG loosing relation to deleted service. - if self.reuse_service: - self.service = self._reuse_service(self.service_name) - if not self.service: - self.service = self._create_service( - self.service_template, - service_name=self.service_name, - namespace_name=self.k8s_namespace.name, - deployment_name=self.deployment_name, - neg_name=self.gcp_neg_name, - test_port=test_port, - ) - - # Create the parentref service - self.frontend_service = self._create_service( - "gamma/frontend_service.yaml", - service_name=self.frontend_service_name, - namespace_name=self.k8s_namespace.name, - ) - - # Create the route. - self.route = self._create_gamma_route( - route_template, - route_name=self.route_name, - service_name=self.service_name, - namespace_name=self.k8s_namespace.name, - test_port=test_port, - frontend_service_name=self.frontend_service_name, - ) - - if self.enable_workload_identity: - # Allow Kubernetes service account to use the GCP service account - # identity. - self._grant_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - - # Create service account - self.service_account = self._create_service_account( - self.service_account_template, - service_account_name=self.service_account_name, - namespace_name=self.k8s_namespace.name, - gcp_service_account=self.gcp_service_account, - ) - - # Always create a new deployment - self.deployment = self._create_deployment( - self.deployment_template, - deployment_name=self.deployment_name, - image_name=self.image_name, - namespace_name=self.k8s_namespace.name, - service_account_name=self.service_account_name, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - replica_count=replica_count, - test_port=test_port, - maintenance_port=maintenance_port, - secure_mode=secure_mode, - bootstrap_version=bootstrap_version, - termination_grace_period_seconds=self.termination_grace_period_seconds, - pre_stop_hook=self.pre_stop_hook, - enable_csm_observability=enable_csm_observability, - ) - - servers = self._make_servers_for_deployment( - replica_count, - test_port=test_port, - maintenance_port=maintenance_port, - log_to_stdout=log_to_stdout, - secure_mode=secure_mode, - ) - - # The controller will not populate the NEGs until there are - # endpoint slices. - # For this reason, we run this check after the servers were created, - # and increase the wait time from 1 minute to 3. - self._wait_service_neg_status_annotation( - self.service_name, - test_port, - timeout_sec=datetime.timedelta(minutes=3).total_seconds(), - ) - - return servers - - def createSessionAffinityPolicy(self, manifest): - self.sa_policy = self._create_session_affinity_policy( - manifest, - session_affinity_policy_name=self.sapolicy_name, - namespace_name=self.k8s_namespace.name, - route_name=self.route_name, - service_name=self.service_name, - ) - - def createSessionAffinityFilter(self): - self.sa_filter = self._create_session_affinity_filter( - "gamma/session_affinity_filter.yaml", - session_affinity_filter_name=self.safilter_name, - namespace_name=self.k8s_namespace.name, - ) - - def createBackendPolicy(self): - self.be_policy = self._create_backend_policy( - "gamma/backend_policy.yaml", - be_policy_name=self.bepolicy_name, - namespace_name=self.k8s_namespace.name, - service_name=self.service_name, - ) - - # pylint: disable=arguments-differ - def cleanup(self, *, force=False, force_namespace=False): - try: - if self.route or force: - self._delete_gamma_route(self.route_name) - self.route = None - - if self.frontend_service or force: - self._delete_service(self.frontend_service_name) - self.frontend_service = None - - if (self.service and not self.reuse_service) or force: - self._delete_service(self.service_name) - self.service = None - - if self.deployment or force: - self._delete_deployment(self.deployment_name) - self.deployment = None - - if self.sa_policy or force: - self._delete_session_affinity_policy(self.sapolicy_name) - self.sa_policy = None - - if self.sa_filter or force: - self._delete_session_affinity_filter(self.safilter_name) - self.sa_filter = None - - if self.be_policy or force: - self._delete_backend_policy(self.bepolicy_name) - self.be_policy = None - - if self.enable_workload_identity and ( - self.service_account or force - ): - self._revoke_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - self._delete_service_account(self.service_account_name) - self.service_account = None - - self._cleanup_namespace(force=(force_namespace and force)) - finally: - self._stop() - - # pylint: enable=arguments-differ diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py deleted file mode 100644 index f4b2685455e..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_base_runner.py +++ /dev/null @@ -1,953 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Common functionality for running xDS Test Client and Server on Kubernetes. -""" -from abc import ABCMeta -import contextlib -import dataclasses -import datetime -import logging -import pathlib -from typing import List, Optional - -import mako.template -import yaml - -from framework.helpers import retryers -import framework.helpers.datetime -import framework.helpers.highlighter -import framework.helpers.rand -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.test_app.runners import base_runner - -logger = logging.getLogger(__name__) - -# Type aliases -_RunnerError = base_runner.RunnerError -_HighlighterYaml = framework.helpers.highlighter.HighlighterYaml -_helper_datetime = framework.helpers.datetime -_datetime = datetime.datetime -_timedelta = datetime.timedelta - - -@dataclasses.dataclass(frozen=True) -class RunHistory: - deployment_id: str - time_start_requested: _datetime - time_start_completed: Optional[_datetime] - time_stopped: _datetime - - -class KubernetesBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): - # Pylint wants abstract classes to override abstract methods. - # pylint: disable=abstract-method - - TEMPLATE_DIR_NAME = "kubernetes-manifests" - TEMPLATE_DIR_RELATIVE_PATH = f"../../../../{TEMPLATE_DIR_NAME}" - ROLE_WORKLOAD_IDENTITY_USER = "roles/iam.workloadIdentityUser" - pod_port_forwarders: List[k8s.PortForwarder] - pod_log_collectors: List[k8s.PodLogCollector] - - # Required fields. - k8s_namespace: k8s.KubernetesNamespace - deployment_name: str - image_name: str - gcp_project: str - gcp_service_account: str - gcp_ui_url: str - - # Fields with default values. - namespace_template: str = "namespace.yaml" - reuse_namespace: bool = False - - # Mutable state. Describes the current run. - namespace: Optional[k8s.V1Namespace] = None - deployment: Optional[k8s.V1Deployment] = None - deployment_id: Optional[str] = None - service_account: Optional[k8s.V1ServiceAccount] = None - time_start_requested: Optional[_datetime] = None - time_start_completed: Optional[_datetime] = None - time_stopped: Optional[_datetime] = None - # The history of all runs performed by this runner. - run_history: List[RunHistory] - - def __init__( - self, - k8s_namespace: k8s.KubernetesNamespace, - *, - deployment_name: str, - image_name: str, - gcp_project: str, - gcp_service_account: str, - gcp_ui_url: str, - namespace_template: Optional[str] = "namespace.yaml", - reuse_namespace: bool = False, - ): - super().__init__() - - # Required fields. - self.deployment_name = deployment_name - self.image_name = image_name - self.gcp_project = gcp_project - # Maps GCP service account to Kubernetes service account - self.gcp_service_account = gcp_service_account - self.gcp_ui_url = gcp_ui_url - - # Kubernetes namespace resources manager. - self.k8s_namespace = k8s_namespace - if namespace_template: - self.namespace_template = namespace_template - self.reuse_namespace = reuse_namespace - - # Mutable state - self.run_history = [] - self.pod_port_forwarders = [] - self.pod_log_collectors = [] - - # Highlighter. - self._highlighter = _HighlighterYaml() - - def run(self, **kwargs): - del kwargs - if not self.time_stopped and self.time_start_requested: - if self.time_start_completed: - raise RuntimeError( - f"Deployment {self.deployment_name}: has already been" - f" started at {self.time_start_completed.isoformat()}" - ) - else: - raise RuntimeError( - f"Deployment {self.deployment_name}: start has already been" - f" requested at {self.time_start_requested.isoformat()}" - ) - - self._reset_state() - self.time_start_requested = _datetime.now() - - self.logs_explorer_link() - if self.reuse_namespace: - self.namespace = self._reuse_namespace() - if not self.namespace: - self.namespace = self._create_namespace( - self.namespace_template, namespace_name=self.k8s_namespace.name - ) - - def _start_completed(self): - self.time_start_completed = _datetime.now() - - def _stop(self): - self.time_stopped = _datetime.now() - if self.time_start_requested and self.deployment_id: - run_history = RunHistory( - deployment_id=self.deployment_id, - time_start_requested=self.time_start_requested, - time_start_completed=self.time_start_completed, - time_stopped=self.time_stopped, - ) - self.run_history.append(run_history) - - def _reset_state(self): - """Reset the mutable state of the previous run.""" - if self.pod_port_forwarders: - logger.warning( - "Port forwarders weren't cleaned up from the past run: %s", - len(self.pod_port_forwarders), - ) - - if self.pod_log_collectors: - logger.warning( - "Pod log collectors weren't cleaned up from the past run: %s", - len(self.pod_log_collectors), - ) - - self.namespace = None - self.deployment = None - self.deployment_id = None - self.service_account = None - self.time_start_requested = None - self.time_start_completed = None - self.time_stopped = None - self.pod_port_forwarders = [] - self.pod_log_collectors = [] - - def _cleanup_namespace(self, *, force=False): - if (self.namespace and not self.reuse_namespace) or force: - self.delete_namespace() - self.namespace = None - - def stop_pod_dependencies(self, *, log_drain_sec: int = 0): - # Signal to stop logging early so less drain time needed. - self.maybe_stop_logging() - - # Stop port forwarders if any. - for pod_port_forwarder in self.pod_port_forwarders: - pod_port_forwarder.close() - self.pod_port_forwarders = [] - - for pod_log_collector in self.pod_log_collectors: - if log_drain_sec > 0 and not pod_log_collector.drain_event.is_set(): - logger.info( - "Draining logs for %s, timeout %i sec", - pod_log_collector.pod_name, - log_drain_sec, - ) - # The close will happen normally at the next message. - pod_log_collector.drain_event.wait(timeout=log_drain_sec) - # Note this will be called from the main thread and may cause - # a race for the log file. Still, at least it'll flush the buffers. - pod_log_collector.flush() - - self.pod_log_collectors = [] - - def get_pod_restarts(self, deployment: k8s.V1Deployment) -> int: - if not self.k8s_namespace or not deployment: - return 0 - total_restart: int = 0 - pods: List[k8s.V1Pod] = self.k8s_namespace.list_deployment_pods( - deployment - ) - for pod in pods: - total_restart += sum( - status.restart_count for status in pod.status.container_statuses - ) - return total_restart - - @classmethod - def _render_template(cls, template_file, **kwargs): - template = mako.template.Template(filename=str(template_file)) - return template.render(**kwargs) - - @classmethod - def _manifests_from_yaml_file(cls, yaml_file): - with open(yaml_file) as f: - with contextlib.closing(yaml.safe_load_all(f)) as yml: - for manifest in yml: - yield manifest - - @classmethod - def _manifests_from_str(cls, document): - with contextlib.closing(yaml.safe_load_all(document)) as yml: - for manifest in yml: - yield manifest - - @classmethod - def _template_file_from_name(cls, template_name): - templates_path = ( - pathlib.Path(__file__).parent / cls.TEMPLATE_DIR_RELATIVE_PATH - ) - return templates_path.joinpath(template_name).resolve() - - def _create_from_template( - self, - template_name, - *, - custom_object: bool = False, - **kwargs, - ) -> object: - template_file = self._template_file_from_name(template_name) - logger.debug("Loading k8s manifest template: %s", template_file) - - yaml_doc = self._render_template(template_file, **kwargs) - logger.info( - "Rendered template %s/%s:\n%s", - self.TEMPLATE_DIR_NAME, - template_name, - self._highlighter.highlight(yaml_doc), - ) - - manifests = self._manifests_from_str(yaml_doc) - manifest = next(manifests) - # Error out on multi-document yaml - if next(manifests, False): - raise _RunnerError( - f"Exactly one document expected in manifest {template_file}" - ) - - k8s_object = self.k8s_namespace.create_single_resource( - manifest, - custom_object=custom_object, - ) - - logger.info("%s %s created", k8s_object.kind, k8s_object.metadata.name) - return k8s_object - - def _reuse_deployment(self, deployment_name) -> k8s.V1Deployment: - deployment = self.k8s_namespace.get_deployment(deployment_name) - # TODO(sergiitk): check if good or must be recreated - return deployment - - def _reuse_service(self, service_name) -> k8s.V1Service: - service = self.k8s_namespace.get_service(service_name) - logger.info("Reusing service: %s", service_name) - # TODO(sergiitk): check if good or must be recreated - return service - - def _reuse_namespace(self) -> k8s.V1Namespace: - logger.info("Reusing namespace: %s", self.k8s_namespace.name) - return self.k8s_namespace.get() - - def _create_pod_monitoring(self, template, **kwargs) -> None: - if not kwargs["namespace_name"]: - raise _RunnerError( - "namespace_name required to create PodMonitoring resource" - ) - if not kwargs["deployment_id"]: - raise _RunnerError( - "deployment_id required to create PodMonitoring resource" - ) - if not kwargs["pod_monitoring_name"]: - raise _RunnerError( - "pod_monitoring_name required to create PodMonitoring resource" - ) - pod_monitoring = self._create_from_template( - template, custom_object=True, **kwargs - ) - if pod_monitoring.metadata.namespace != kwargs["namespace_name"]: - raise _RunnerError( - "PodMonitoring resource created with unexpected namespace: " - f"{pod_monitoring.metadata.namespace}" - ) - logger.debug( - "PodMonitoring %s created at %s", - pod_monitoring.metadata.name, - pod_monitoring.metadata.creation_timestamp, - ) - return pod_monitoring - - def _create_namespace(self, template, **kwargs) -> k8s.V1Namespace: - namespace = self._create_from_template(template, **kwargs) - if not isinstance(namespace, k8s.V1Namespace): - raise _RunnerError( - f"Expected V1Namespace to be created from manifest {template}" - ) - if namespace.metadata.name != kwargs["namespace_name"]: - raise _RunnerError( - "V1Namespace created with unexpected name: " - f"{namespace.metadata.name}" - ) - logger.debug( - "V1Namespace %s created at %s", - namespace.metadata.self_link, - namespace.metadata.creation_timestamp, - ) - return namespace - - @classmethod - def _get_workload_identity_member_name( - cls, project, namespace_name, service_account_name - ): - """ - Returns workload identity member name used to authenticate Kubernetes - service accounts. - - https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity - """ - return ( - f"serviceAccount:{project}.svc.id.goog" - f"[{namespace_name}/{service_account_name}]" - ) - - def _grant_workload_identity_user( - self, *, gcp_iam, gcp_service_account, service_account_name - ): - workload_identity_member = self._get_workload_identity_member_name( - gcp_iam.project, self.k8s_namespace.name, service_account_name - ) - logger.info( - "Granting %s to %s for GCP Service Account %s", - self.ROLE_WORKLOAD_IDENTITY_USER, - workload_identity_member, - gcp_service_account, - ) - - gcp_iam.add_service_account_iam_policy_binding( - gcp_service_account, - self.ROLE_WORKLOAD_IDENTITY_USER, - workload_identity_member, - ) - - def _revoke_workload_identity_user( - self, *, gcp_iam, gcp_service_account, service_account_name - ): - workload_identity_member = self._get_workload_identity_member_name( - gcp_iam.project, self.k8s_namespace.name, service_account_name - ) - logger.info( - "Revoking %s from %s for GCP Service Account %s", - self.ROLE_WORKLOAD_IDENTITY_USER, - workload_identity_member, - gcp_service_account, - ) - try: - gcp_iam.remove_service_account_iam_policy_binding( - gcp_service_account, - self.ROLE_WORKLOAD_IDENTITY_USER, - workload_identity_member, - ) - except gcp.api.Error as error: - logger.warning( - "Failed %s from %s for Service Account %s: %r", - self.ROLE_WORKLOAD_IDENTITY_USER, - workload_identity_member, - gcp_service_account, - error, - ) - - def _create_service_account( - self, template, **kwargs - ) -> k8s.V1ServiceAccount: - resource = self._create_from_template(template, **kwargs) - if not isinstance(resource, k8s.V1ServiceAccount): - raise _RunnerError( - "Expected V1ServiceAccount to be created " - f"from manifest {template}" - ) - if resource.metadata.name != kwargs["service_account_name"]: - raise _RunnerError( - "V1ServiceAccount created with unexpected name: " - f"{resource.metadata.name}" - ) - logger.debug( - "V1ServiceAccount %s created at %s", - resource.metadata.self_link, - resource.metadata.creation_timestamp, - ) - return resource - - def delete_pod_async(self, pod_name: str): - logger.info( - "Initiating deletion of pod %s in namespace %s", - pod_name, - self.k8s_namespace.name, - ) - self.k8s_namespace.delete_pod_async(pod_name) - - def _create_deployment(self, template, **kwargs) -> k8s.V1Deployment: - # Not making deployment_name an explicit kwarg to be consistent with - # the rest of the _create_* methods, which pass kwargs as-is - # to _create_from_template(), so that the kwargs dict is unpacked into - # template variables and their values. - if "deployment_name" not in kwargs: - raise TypeError( - "Missing required keyword-only argument: deployment_name" - ) - - # Automatically apply random deployment_id to use in the matchLabels - # to prevent selecting pods in the same namespace belonging to - # a different deployment. - if "deployment_id" not in kwargs: - rand_id: str = framework.helpers.rand.rand_string(lowercase=True) - # Fun edge case: when rand_string() happen to generate numbers only, - # yaml interprets deployment_id label value as an integer, - # but k8s expects label values to be strings. Lol. K8s responds - # with a barely readable 400 Bad Request error: 'ReadString: expects - # \" or n, but found 9, error found in #10 byte of ...|ent_id'. - # Prepending deployment name forces deployment_id into a string, - # as well as it's just a better description. - self.deployment_id = f'{kwargs["deployment_name"]}-{rand_id}' - kwargs["deployment_id"] = self.deployment_id - else: - self.deployment_id = kwargs["deployment_id"] - - deployment = self._create_from_template(template, **kwargs) - if not isinstance(deployment, k8s.V1Deployment): - raise _RunnerError( - f"Expected V1Deployment to be created from manifest {template}" - ) - if deployment.metadata.name != kwargs["deployment_name"]: - raise _RunnerError( - "V1Deployment created with unexpected name: " - f"{deployment.metadata.name}" - ) - logger.debug( - "V1Deployment %s created at %s", - deployment.metadata.self_link, - deployment.metadata.creation_timestamp, - ) - return deployment - - def _create_gamma_route(self, template, **kwargs) -> k8s.GammaHttpRoute: - route = self._create_from_template( - template, - custom_object=True, - **kwargs, - ) - if not ( - isinstance(route, k8s.GammaHttpRoute) and route.kind == "HTTPRoute" - ): - raise _RunnerError( - f"Expected ResourceInstance[HTTPRoute] to be created from" - f" manifest {template}" - ) - if route.metadata.name != kwargs["route_name"]: - raise _RunnerError( - "ResourceInstance[HTTPRoute] created with unexpected name: " - f"{route.metadata.name}" - ) - logger.debug( - "ResourceInstance[HTTPRoute] %s created at %s", - route.metadata.name, - route.metadata.creation_timestamp, - ) - return route - - def _create_session_affinity_policy( - self, template, **kwargs - ) -> k8s.GcpSessionAffinityPolicy: - saPolicy = self._create_from_template( - template, - custom_object=True, - **kwargs, - ) - if not ( - isinstance(saPolicy, k8s.GcpSessionAffinityPolicy) - and saPolicy.kind == "GCPSessionAffinityPolicy" - ): - raise _RunnerError( - f"Expected ResourceInstance[GCPSessionAffinityPolicy] to be" - f" created from manifest {template}" - ) - if saPolicy.metadata.name != kwargs["session_affinity_policy_name"]: - raise _RunnerError( - "ResourceInstance[GCPSessionAffinityPolicy] created with" - f" unexpected name: {saPolicy.metadata.name}" - ) - logger.debug( - "ResourceInstance[GCPSessionAffinityPolicy] %s created at %s", - saPolicy.metadata.name, - saPolicy.metadata.creation_timestamp, - ) - return saPolicy - - def _create_session_affinity_filter( - self, template, **kwargs - ) -> k8s.GcpSessionAffinityFilter: - saFilter = self._create_from_template( - template, - custom_object=True, - **kwargs, - ) - if not ( - isinstance(saFilter, k8s.GcpSessionAffinityFilter) - and saFilter.kind == "GCPSessionAffinityFilter" - ): - raise _RunnerError( - f"Expected ResourceInstance[GCPSessionAffinityFilter] to be" - f" created from manifest {template}" - ) - if saFilter.metadata.name != kwargs["session_affinity_filter_name"]: - raise _RunnerError( - "ResourceInstance[GCPSessionAffinityFilter] created with" - f" unexpected name: {saFilter.metadata.name}" - ) - logger.debug( - "ResourceInstance[GCPSessionAffinityFilter] %s created at %s", - saFilter.metadata.name, - saFilter.metadata.creation_timestamp, - ) - return saFilter - - def _create_backend_policy( - self, template, **kwargs - ) -> k8s.GcpBackendPolicy: - be_policy = self._create_from_template( - template, - custom_object=True, - **kwargs, - ) - if not ( - isinstance(be_policy, k8s.GcpBackendPolicy) - and be_policy.kind == "GCPBackendPolicy" - ): - raise _RunnerError( - f"Expected ResourceInstance[GCPBackendPolicy] to be" - f" created from manifest {template}" - ) - if be_policy.metadata.name != kwargs["be_policy_name"]: - raise _RunnerError( - "ResourceInstance[GCPBackendPolicy] created with" - f" unexpected name: {be_policy.metadata.name}" - ) - logger.debug( - "ResourceInstance[GCPBackendPolicy] %s created at %s", - be_policy.metadata.name, - be_policy.metadata.creation_timestamp, - ) - return be_policy - - def _create_service(self, template, **kwargs) -> k8s.V1Service: - service = self._create_from_template(template, **kwargs) - if not isinstance(service, k8s.V1Service): - raise _RunnerError( - f"Expected V1Service to be created from manifest {template}" - ) - if service.metadata.name != kwargs["service_name"]: - raise _RunnerError( - "V1Service created with unexpected name: " - f"{service.metadata.name}" - ) - logger.debug( - "V1Service %s created at %s", - service.metadata.self_link, - service.metadata.creation_timestamp, - ) - return service - - def _delete_gamma_route(self, name, wait_for_deletion=True): - logger.info("Deleting HTTPRoute %s", name) - try: - self.k8s_namespace.delete_gamma_route(name) - except k8s.NotFound: - logger.debug( - "HTTPRoute %s not deleted since it doesn't exist", name - ) - return - except retryers.RetryError as e: - logger.warning("HTTPRoute %s deletion failed: %s", name, e) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_get_gamma_route_deleted(name) - logger.info("HTTPRoute %s deleted", name) - - def _delete_session_affinity_policy(self, name, wait_for_deletion=True): - logger.info("Deleting GCPSessionAffinityPolicy %s", name) - try: - self.k8s_namespace.delete_session_affinity_policy(name) - except k8s.NotFound: - logger.debug( - "GCPSessionAffinityPolicy %s not deleted since it" - " doesn't exist", - name, - ) - return - except retryers.RetryError as e: - logger.warning( - "GCPSessionAffinityPolicy %s deletion failed: %s", name, e - ) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_get_session_affinity_policy_deleted( - name - ) - logger.info("GCPSessionAffinityPolicy %s deleted", name) - - def _delete_session_affinity_filter(self, name, wait_for_deletion=True): - logger.info("Deleting GCPSessionAffinityFilter %s", name) - try: - self.k8s_namespace.delete_session_affinity_filter(name) - except k8s.NotFound: - logger.debug( - "GCPSessionAffinityFilter %s not deleted since it" - " doesn't exist", - name, - ) - return - except retryers.RetryError as e: - logger.warning( - "GCPSessionAffinityFilter %s deletion failed: %s", name, e - ) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_get_session_affinity_filter_deleted( - name - ) - logger.info("GCPSessionAffinityFilter %s deleted", name) - - def _delete_backend_policy(self, name, wait_for_deletion=True): - logger.info("Deleting GCPBackendPolicy %s", name) - try: - self.k8s_namespace.delete_backend_policy(name) - except k8s.NotFound: - logger.debug( - "GGCPBackendPolicy %s not deleted since it doesn't exist", name - ) - return - except retryers.RetryError as e: - logger.warning("GGCPBackendPolicy %s deletion failed: %s", name, e) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_get_backend_policy_deleted(name) - logger.info("GCPBackendPolicy %s deleted", name) - - def _delete_deployment(self, name, wait_for_deletion=True): - logger.info("Deleting deployment %s", name) - self.stop_pod_dependencies() - try: - self.k8s_namespace.delete_deployment(name) - except k8s.NotFound: - logger.debug( - "Deployment %s not deleted since it doesn't exist", name - ) - return - except retryers.RetryError as e: - logger.warning("Deployment %s deletion failed: %s", name, e) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_deployment_deleted(name) - logger.info("Deployment %s deleted", name) - - def _delete_service(self, name, wait_for_deletion=True): - logger.info("Deleting service %s", name) - try: - self.k8s_namespace.delete_service(name) - except k8s.NotFound: - logger.debug("Service %s not deleted since it doesn't exist", name) - return - except retryers.RetryError as e: - logger.warning("Service %s deletion failed: %s", name, e) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_service_deleted(name) - - logger.info("Service %s deleted", name) - - def _delete_service_account(self, name, wait_for_deletion=True): - logger.info("Deleting service account %s", name) - try: - self.k8s_namespace.delete_service_account(name) - except k8s.NotFound: - logger.debug( - "Service account %s not deleted since it doesn't exist", name - ) - return - except retryers.RetryError as e: - logger.warning("Service account %s deletion failed: %s", name, e) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_service_account_deleted(name) - logger.info("Service account %s deleted", name) - - def delete_namespace(self, wait_for_deletion=True): - logger.info("Deleting namespace %s", self.k8s_namespace.name) - try: - self.k8s_namespace.delete() - except k8s.NotFound: - logger.debug( - "Namespace %s not deleted since it doesn't exist", - self.k8s_namespace.name, - ) - return - except retryers.RetryError as e: - logger.warning( - "Namespace %s deletion failed: %s", self.k8s_namespace.name, e - ) - return - - if wait_for_deletion: - self.k8s_namespace.wait_for_namespace_deleted() - logger.info("Namespace %s deleted", self.k8s_namespace.name) - - def _wait_deployment_with_available_replicas(self, name, count=1, **kwargs): - logger.info( - "Waiting for deployment %s to report %s available replica(s)", - name, - count, - ) - self.k8s_namespace.wait_for_deployment_available_replicas( - name, count, **kwargs - ) - deployment = self.k8s_namespace.get_deployment(name) - logger.info( - "Deployment %s has %i replicas available", - deployment.metadata.name, - deployment.status.available_replicas, - ) - - def _wait_deployment_pod_count( - self, deployment: k8s.V1Deployment, count: int = 1, **kwargs - ) -> List[str]: - logger.info( - "Waiting for deployment %s to initialize %s pod(s)", - deployment.metadata.name, - count, - ) - self.k8s_namespace.wait_for_deployment_replica_count( - deployment, count, **kwargs - ) - pods = self.k8s_namespace.list_deployment_pods(deployment) - pod_names = [pod.metadata.name for pod in pods] - logger.info( - "Deployment %s initialized %i pod(s): %s", - deployment.metadata.name, - count, - pod_names, - ) - # Pods may not be started yet, just return the names. - return pod_names - - def _wait_pod_started(self, name, **kwargs) -> k8s.V1Pod: - logger.info("Waiting for pod %s to start", name) - self.k8s_namespace.wait_for_pod_started(name, **kwargs) - pod = self.k8s_namespace.get_pod(name) - logger.info( - "Pod %s ready, IP: %s", pod.metadata.name, pod.status.pod_ip - ) - return pod - - def _start_port_forwarding_pod( - self, pod: k8s.V1Pod, remote_port: int - ) -> k8s.PortForwarder: - logger.info( - "LOCAL DEV MODE: Enabling port forwarding to %s:%s", - pod.status.pod_ip, - remote_port, - ) - port_forwarder = self.k8s_namespace.port_forward_pod(pod, remote_port) - self.pod_port_forwarders.append(port_forwarder) - return port_forwarder - - def _start_logging_pod( - self, pod: k8s.V1Pod, *, log_to_stdout: bool = False - ) -> k8s.PodLogCollector: - pod_name = pod.metadata.name - logfile_name = f"{self.k8s_namespace.name}_{pod_name}.log" - log_path = self.logs_subdir / logfile_name - logger.info( - "Enabling log collection from pod %s to %s", - pod_name, - log_path.relative_to(self.logs_subdir.parent.parent), - ) - pod_log_collector = self.k8s_namespace.pod_start_logging( - pod_name=pod_name, - log_path=log_path, - log_stop_event=self.log_stop_event, - log_to_stdout=log_to_stdout, - # Timestamps are enabled because not all language implementations - # include them. - # TODO(sergiitk): Make this setting language-specific. - log_timestamps=True, - ) - self.pod_log_collectors.append(pod_log_collector) - return pod_log_collector - - def _wait_service_neg_status_annotation( - self, - service_name: str, - service_port: int, - **kwargs, - ) -> None: - logger.info( - "Waiting for '%s' annotation for a NEG at port %s to be assigned to" - " Kubernetes Service %s in namespace %s", - self.k8s_namespace.NEG_STATUS_ANNOTATION, - service_port, - service_name, - self.k8s_namespace.name, - ) - self.k8s_namespace.wait_for_service_neg_status_annotation( - service_name, **kwargs - ) - neg_name, neg_zones = self.k8s_namespace.parse_service_neg_status( - service_name, service_port - ) - logger.info( - "Detected '%s' annotation for Kubernetes Service %s, namespace %s:" - " neg_name=%s, port=%s, zones=%s", - self.k8s_namespace.NEG_STATUS_ANNOTATION, - service_name, - self.k8s_namespace.name, - neg_name, - service_port, - neg_zones, - ) - - def logs_explorer_link(self): - """Prints GCP Logs Explorer link to all runs of the deployment.""" - self._logs_explorer_link( - deployment_name=self.deployment_name, - namespace_name=self.k8s_namespace.name, - gcp_project=self.gcp_project, - gcp_ui_url=self.gcp_ui_url, - ) - - def logs_explorer_run_history_links(self): - """Prints a separate GCP Logs Explorer link for each run *completed* by - the runner. - - This excludes the current run, if it hasn't been completed. - """ - if not self.run_history: - logger.info("No completed deployments of %s", self.deployment_name) - return - for run in self.run_history: - self._logs_explorer_link( - deployment_name=self.deployment_name, - namespace_name=self.k8s_namespace.name, - gcp_project=self.gcp_project, - gcp_ui_url=self.gcp_ui_url, - deployment_id=run.deployment_id, - start_time=run.time_start_requested, - end_time=run.time_stopped, - ) - - @classmethod - def _logs_explorer_link( - cls, - *, - deployment_name: str, - namespace_name: str, - gcp_project: str, - gcp_ui_url: str, - deployment_id: Optional[str] = None, - start_time: Optional[_datetime] = None, - end_time: Optional[_datetime] = None, - ): - """Output the link to test server/client logs in GCP Logs Explorer.""" - if not start_time: - start_time = _datetime.now() - if not end_time: - end_time = start_time + _timedelta(minutes=30) - - logs_start = _helper_datetime.iso8601_utc_time(start_time) - logs_end = _helper_datetime.iso8601_utc_time(end_time) - request = {"timeRange": f"{logs_start}/{logs_end}"} - query = { - "resource.type": "k8s_container", - "resource.labels.project_id": gcp_project, - "resource.labels.container_name": deployment_name, - "resource.labels.namespace_name": namespace_name, - } - if deployment_id: - query['labels."k8s-pod/deployment_id"'] = deployment_id - - link = cls._logs_explorer_link_from_params( - gcp_ui_url=gcp_ui_url, - gcp_project=gcp_project, - query=query, - request=request, - ) - link_to = deployment_id if deployment_id else deployment_name - # A whitespace at the end to indicate the end of the url. - logger.info("GCP Logs Explorer link to %s:\n%s ", link_to, link) - - @classmethod - def _make_namespace_name( - cls, resource_prefix: str, resource_suffix: str, name: str - ) -> str: - """A helper to make consistent test app kubernetes namespace name - for given resource prefix and suffix.""" - parts = [resource_prefix, name] - # Avoid trailing dash when the suffix is empty. - if resource_suffix: - parts.append(resource_suffix) - return "-".join(parts) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_client_runner.py deleted file mode 100644 index dd26166ee31..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ /dev/null @@ -1,238 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Run xDS Test Client on Kubernetes. -""" -import logging -from typing import Optional - -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.test_app.client_app import XdsTestClient -from framework.test_app.runners.k8s import k8s_base_runner - -logger = logging.getLogger(__name__) - - -class KubernetesClientRunner(k8s_base_runner.KubernetesBaseRunner): - # Required fields. - xds_server_uri: str - stats_port: int - deployment_template: str - enable_workload_identity: bool - debug_use_port_forwarding: bool - td_bootstrap_image: str - network: str - - # Optional fields. - service_account_name: Optional[str] = None - service_account_template: Optional[str] = None - gcp_iam: Optional[gcp.iam.IamV1] = None - - def __init__( # pylint: disable=too-many-locals - self, - k8s_namespace: k8s.KubernetesNamespace, - *, - deployment_name: str, - image_name: str, - td_bootstrap_image: str, - network="default", - xds_server_uri: Optional[str] = None, - gcp_api_manager: gcp.api.GcpApiManager, - gcp_project: str, - gcp_service_account: str, - service_account_name: Optional[str] = None, - stats_port: int = 8079, - deployment_template: str = "client.deployment.yaml", - service_account_template: str = "service-account.yaml", - reuse_namespace: bool = False, - namespace_template: Optional[str] = None, - debug_use_port_forwarding: bool = False, - enable_workload_identity: bool = True, - ): - super().__init__( - k8s_namespace, - deployment_name=deployment_name, - image_name=image_name, - gcp_project=gcp_project, - gcp_service_account=gcp_service_account, - gcp_ui_url=gcp_api_manager.gcp_ui_url, - namespace_template=namespace_template, - reuse_namespace=reuse_namespace, - ) - - # Settings - self.stats_port = stats_port - self.deployment_template = deployment_template - self.enable_workload_identity = enable_workload_identity - self.debug_use_port_forwarding = debug_use_port_forwarding - - # Used by the TD bootstrap generator. - self.td_bootstrap_image = td_bootstrap_image - self.network = network - self.xds_server_uri = xds_server_uri - - # Workload identity settings: - if self.enable_workload_identity: - # Kubernetes service account. - self.service_account_name = service_account_name or deployment_name - self.service_account_template = service_account_template - # GCP IAM API used to grant allow workload service accounts - # permission to use GCP service account identity. - self.gcp_iam = gcp.iam.IamV1(gcp_api_manager, gcp_project) - - def run( # pylint: disable=arguments-differ - self, - *, - server_target, - rpc="UnaryCall", - qps=25, - metadata="", - secure_mode=False, - config_mesh=None, - generate_mesh_id=False, - print_response=False, - log_to_stdout: bool = False, - enable_csm_observability: bool = False, - ) -> XdsTestClient: - logger.info( - ( - 'Deploying xDS test client "%s" to k8s namespace %s: ' - "server_target=%s rpc=%s qps=%s metadata=%r secure_mode=%s " - "print_response=%s" - ), - self.deployment_name, - self.k8s_namespace.name, - server_target, - rpc, - qps, - metadata, - secure_mode, - print_response, - ) - super().run() - - if self.enable_workload_identity: - # Allow Kubernetes service account to use the GCP service account - # identity. - self._grant_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - - # Create service account - self.service_account = self._create_service_account( - self.service_account_template, - service_account_name=self.service_account_name, - namespace_name=self.k8s_namespace.name, - gcp_service_account=self.gcp_service_account, - ) - - # Always create a new deployment - self.deployment = self._create_deployment( - self.deployment_template, - deployment_name=self.deployment_name, - image_name=self.image_name, - namespace_name=self.k8s_namespace.name, - service_account_name=self.service_account_name, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - stats_port=self.stats_port, - server_target=server_target, - rpc=rpc, - qps=qps, - metadata=metadata, - secure_mode=secure_mode, - config_mesh=config_mesh, - generate_mesh_id=generate_mesh_id, - print_response=print_response, - enable_csm_observability=enable_csm_observability, - ) - - # Create a PodMonitoring resource if CSM Observability is enabled - # This is GMP (Google Managed Prometheus) - if enable_csm_observability: - self._create_pod_monitoring( - "csm/pod-monitoring.yaml", - namespace_name=self.k8s_namespace.name, - deployment_id=self.deployment_id, - pod_monitoring_name="%s-gmp" % self.deployment_id, - ) - - # Load test client pod. We need only one client at the moment - pod_name = self._wait_deployment_pod_count(self.deployment)[0] - pod: k8s.V1Pod = self._wait_pod_started(pod_name) - if self.should_collect_logs: - self._start_logging_pod(pod, log_to_stdout=log_to_stdout) - - # Verify the deployment reports all pods started as well. - self._wait_deployment_with_available_replicas(self.deployment_name) - self._start_completed() - - return self._xds_test_client_for_pod(pod, server_target=server_target) - - def _xds_test_client_for_pod( - self, pod: k8s.V1Pod, *, server_target: str - ) -> XdsTestClient: - if self.debug_use_port_forwarding: - pf = self._start_port_forwarding_pod(pod, self.stats_port) - rpc_port, rpc_host = pf.local_port, pf.local_address - else: - rpc_port, rpc_host = self.stats_port, None - - return XdsTestClient( - ip=pod.status.pod_ip, - rpc_port=rpc_port, - server_target=server_target, - hostname=pod.metadata.name, - rpc_host=rpc_host, - ) - - # pylint: disable=arguments-differ - def cleanup(self, *, force=False, force_namespace=False): - # TODO(sergiitk): rename to stop(). - try: - if self.deployment or force: - self._delete_deployment(self.deployment_name) - self.deployment = None - if self.enable_workload_identity and ( - self.service_account or force - ): - self._revoke_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - self._delete_service_account(self.service_account_name) - self.service_account = None - self._cleanup_namespace(force=force_namespace and force) - finally: - self._stop() - - # pylint: enable=arguments-differ - - @classmethod - def make_namespace_name( - cls, resource_prefix: str, resource_suffix: str, name: str = "client" - ) -> str: - """A helper to make consistent XdsTestClient kubernetes namespace name - for given resource prefix and suffix. - - Note: the idea is to intentionally produce different namespace name for - the test server, and the test client, as that closely mimics real-world - deployments. - """ - return cls._make_namespace_name(resource_prefix, resource_suffix, name) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_server_runner.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_server_runner.py deleted file mode 100644 index 90615fd9ae7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/runners/k8s/k8s_xds_server_runner.py +++ /dev/null @@ -1,324 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Run xDS Test Client on Kubernetes. -""" -import logging -from typing import List, Optional - -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.test_app.runners.k8s import k8s_base_runner -from framework.test_app.server_app import XdsTestServer - -logger = logging.getLogger(__name__) - - -class KubernetesServerRunner(k8s_base_runner.KubernetesBaseRunner): - DEFAULT_TEST_PORT = 8080 - DEFAULT_MAINTENANCE_PORT = 8080 - DEFAULT_SECURE_MODE_MAINTENANCE_PORT = 8081 - - # Required fields. - deployment_template: str - service_name: str - service_template: str - reuse_service: bool - enable_workload_identity: bool - debug_use_port_forwarding: bool - gcp_neg_name: str - td_bootstrap_image: str - xds_server_uri: str - network: str - - # Optional fields. - service_account_name: Optional[str] = None - service_account_template: Optional[str] = None - gcp_iam: Optional[gcp.iam.IamV1] = None - - # Mutable state. - service: Optional[k8s.V1Service] = None - - def __init__( # pylint: disable=too-many-locals - self, - k8s_namespace: k8s.KubernetesNamespace, - *, - deployment_name: str, - image_name: str, - td_bootstrap_image: str, - network: str = "default", - xds_server_uri: Optional[str] = None, - gcp_api_manager: gcp.api.GcpApiManager, - gcp_project: str, - gcp_service_account: str, - service_account_name: Optional[str] = None, - service_name: Optional[str] = None, - neg_name: Optional[str] = None, - deployment_template: str = "server.deployment.yaml", - service_account_template: str = "service-account.yaml", - service_template: str = "server.service.yaml", - reuse_service: bool = False, - reuse_namespace: bool = False, - namespace_template: Optional[str] = None, - debug_use_port_forwarding: bool = False, - enable_workload_identity: bool = True, - ): - super().__init__( - k8s_namespace, - deployment_name=deployment_name, - image_name=image_name, - gcp_project=gcp_project, - gcp_service_account=gcp_service_account, - gcp_ui_url=gcp_api_manager.gcp_ui_url, - namespace_template=namespace_template, - reuse_namespace=reuse_namespace, - ) - - # Settings - self.deployment_template = deployment_template - self.service_name = service_name or deployment_name - self.service_template = service_template - self.reuse_service = reuse_service - self.enable_workload_identity = enable_workload_identity - self.debug_use_port_forwarding = debug_use_port_forwarding - # GCP Network Endpoint Group. - self.gcp_neg_name = neg_name or ( - f"{self.k8s_namespace.name}-{self.service_name}" - ) - - # Used by the TD bootstrap generator. - self.td_bootstrap_image = td_bootstrap_image - self.network = network - self.xds_server_uri = xds_server_uri - - # Workload identity settings: - if self.enable_workload_identity: - # Kubernetes service account. - self.service_account_name = service_account_name or deployment_name - self.service_account_template = service_account_template - # GCP IAM API used to grant allow workload service accounts - # permission to use GCP service account identity. - self.gcp_iam = gcp.iam.IamV1(gcp_api_manager, gcp_project) - - def run( # pylint: disable=arguments-differ,too-many-branches - self, - *, - test_port: int = DEFAULT_TEST_PORT, - maintenance_port: Optional[int] = None, - secure_mode: bool = False, - replica_count: int = 1, - log_to_stdout: bool = False, - bootstrap_version: Optional[str] = None, - ) -> List[XdsTestServer]: - if not maintenance_port: - maintenance_port = self._get_default_maintenance_port(secure_mode) - - # Implementation detail: in secure mode, maintenance ("backchannel") - # port must be different from the test port so communication with - # maintenance services can be reached independently of the security - # configuration under test. - if secure_mode and maintenance_port == test_port: - raise ValueError( - "port and maintenance_port must be different " - "when running test server in secure mode" - ) - # To avoid bugs with comparing wrong types. - if not ( - isinstance(test_port, int) and isinstance(maintenance_port, int) - ): - raise TypeError("Port numbers must be integer") - - if secure_mode and not self.enable_workload_identity: - raise ValueError("Secure mode requires Workload Identity enabled.") - - logger.info( - ( - 'Deploying xDS test server "%s" to k8s namespace %s:' - " test_port=%s maintenance_port=%s secure_mode=%s" - " replica_count=%s" - ), - self.deployment_name, - self.k8s_namespace.name, - test_port, - maintenance_port, - secure_mode, - replica_count, - ) - super().run() - - # Reuse existing if requested, create a new deployment when missing. - # Useful for debugging to avoid NEG loosing relation to deleted service. - if self.reuse_service: - self.service = self._reuse_service(self.service_name) - if not self.service: - self.service = self._create_service( - self.service_template, - service_name=self.service_name, - namespace_name=self.k8s_namespace.name, - deployment_name=self.deployment_name, - neg_name=self.gcp_neg_name, - test_port=test_port, - ) - self._wait_service_neg_status_annotation(self.service_name, test_port) - - if self.enable_workload_identity: - # Allow Kubernetes service account to use the GCP service account - # identity. - self._grant_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - - # Create service account - self.service_account = self._create_service_account( - self.service_account_template, - service_account_name=self.service_account_name, - namespace_name=self.k8s_namespace.name, - gcp_service_account=self.gcp_service_account, - ) - - # Always create a new deployment - self.deployment = self._create_deployment( - self.deployment_template, - deployment_name=self.deployment_name, - image_name=self.image_name, - namespace_name=self.k8s_namespace.name, - service_account_name=self.service_account_name, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - replica_count=replica_count, - test_port=test_port, - maintenance_port=maintenance_port, - secure_mode=secure_mode, - bootstrap_version=bootstrap_version, - ) - - return self._make_servers_for_deployment( - replica_count, - test_port=test_port, - maintenance_port=maintenance_port, - log_to_stdout=log_to_stdout, - secure_mode=secure_mode, - ) - - def _make_servers_for_deployment( - self, - replica_count, - *, - test_port: int, - maintenance_port: int, - log_to_stdout: bool, - secure_mode: bool = False, - ) -> List[XdsTestServer]: - pod_names = self._wait_deployment_pod_count( - self.deployment, replica_count - ) - pods = [] - for pod_name in pod_names: - pod = self._wait_pod_started(pod_name) - pods.append(pod) - if self.should_collect_logs: - self._start_logging_pod(pod, log_to_stdout=log_to_stdout) - - # Verify the deployment reports all pods started as well. - self._wait_deployment_with_available_replicas( - self.deployment_name, replica_count - ) - self._start_completed() - - servers: List[XdsTestServer] = [] - for pod in pods: - servers.append( - self._xds_test_server_for_pod( - pod, - test_port=test_port, - maintenance_port=maintenance_port, - secure_mode=secure_mode, - ) - ) - return servers - - def _get_default_maintenance_port(self, secure_mode: bool) -> int: - if not secure_mode: - maintenance_port = self.DEFAULT_MAINTENANCE_PORT - else: - maintenance_port = self.DEFAULT_SECURE_MODE_MAINTENANCE_PORT - return maintenance_port - - def _xds_test_server_for_pod( - self, - pod: k8s.V1Pod, - *, - test_port: int = DEFAULT_TEST_PORT, - maintenance_port: Optional[int] = None, - secure_mode: bool = False, - ) -> XdsTestServer: - if maintenance_port is None: - maintenance_port = self._get_default_maintenance_port(secure_mode) - - if self.debug_use_port_forwarding: - pf = self._start_port_forwarding_pod(pod, maintenance_port) - rpc_port, rpc_host = pf.local_port, pf.local_address - else: - rpc_port, rpc_host = maintenance_port, None - - return XdsTestServer( - ip=pod.status.pod_ip, - rpc_port=test_port, - hostname=pod.metadata.name, - maintenance_port=rpc_port, - secure_mode=secure_mode, - rpc_host=rpc_host, - ) - - # pylint: disable=arguments-differ - def cleanup(self, *, force=False, force_namespace=False): - # TODO(sergiitk): rename to stop(). - try: - if self.deployment or force: - self._delete_deployment(self.deployment_name) - self.deployment = None - if (self.service and not self.reuse_service) or force: - self._delete_service(self.service_name) - self.service = None - if self.enable_workload_identity and ( - self.service_account or force - ): - self._revoke_workload_identity_user( - gcp_iam=self.gcp_iam, - gcp_service_account=self.gcp_service_account, - service_account_name=self.service_account_name, - ) - self._delete_service_account(self.service_account_name) - self.service_account = None - self._cleanup_namespace(force=(force_namespace and force)) - finally: - self._stop() - - # pylint: enable=arguments-differ - - @classmethod - def make_namespace_name( - cls, resource_prefix: str, resource_suffix: str, name: str = "server" - ) -> str: - """A helper to make consistent XdsTestServer kubernetes namespace name - for given resource prefix and suffix. - - Note: the idea is to intentionally produce different namespace name for - the test server, and the test client, as that closely mimics real-world - deployments. - """ - return cls._make_namespace_name(resource_prefix, resource_suffix, name) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_app/server_app.py b/tools/run_tests/xds_k8s_test_driver/framework/test_app/server_app.py deleted file mode 100644 index 57444b73c7d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_app/server_app.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Provides an interface to xDS Test Server running remotely. -""" -import functools -import logging -from typing import Iterator, Optional - -import framework.rpc -from framework.rpc import grpc_channelz -from framework.rpc import grpc_testing - -logger = logging.getLogger(__name__) - -# Type aliases -_ChannelzServiceClient = grpc_channelz.ChannelzServiceClient -_XdsUpdateHealthServiceClient = grpc_testing.XdsUpdateHealthServiceClient -_HealthClient = grpc_testing.HealthClient - - -class XdsTestServer(framework.rpc.grpc.GrpcApp): - """ - Represents RPC services implemented in Server component of the xDS test app. - https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#server - """ - - # A unique host name identifying each server replica. - # Server implementation must return this in the SimpleResponse.hostname, - # which client uses as the key in rpcs_by_peer map. - hostname: str - - def __init__( - self, - *, - ip: str, - rpc_port: int, - hostname: str, - maintenance_port: Optional[int] = None, - secure_mode: Optional[bool] = False, - xds_host: Optional[str] = None, - xds_port: Optional[int] = None, - rpc_host: Optional[str] = None, - ): - super().__init__(rpc_host=(rpc_host or ip)) - self.ip = ip - self.rpc_port = rpc_port - self.hostname = hostname - self.maintenance_port = maintenance_port or rpc_port - self.secure_mode = secure_mode - self.xds_host, self.xds_port = xds_host, xds_port - - @property - @functools.lru_cache(None) - def channelz(self) -> _ChannelzServiceClient: - return _ChannelzServiceClient( - self._make_channel(self.maintenance_port), - log_target=f"{self.hostname}:{self.maintenance_port}", - ) - - @property - @functools.lru_cache(None) - def update_health_service_client(self) -> _XdsUpdateHealthServiceClient: - return _XdsUpdateHealthServiceClient( - self._make_channel(self.maintenance_port), - log_target=f"{self.hostname}:{self.maintenance_port}", - ) - - @property - @functools.lru_cache(None) - def health_client(self) -> _HealthClient: - return _HealthClient( - self._make_channel(self.maintenance_port), - log_target=f"{self.hostname}:{self.maintenance_port}", - ) - - def set_serving(self): - logger.info("[%s] >> Setting health status to SERVING", self.hostname) - self.update_health_service_client.set_serving() - logger.info( - "[%s] << Health status %s", - self.hostname, - self.health_client.check_health(), - ) - - def set_not_serving(self): - logger.info( - "[%s] >> Setting health status to NOT_SERVING", self.hostname - ) - self.update_health_service_client.set_not_serving() - logger.info( - "[%s] << Health status %s", - self.hostname, - self.health_client.check_health(), - ) - - def set_xds_address(self, xds_host, xds_port: Optional[int] = None): - self.xds_host, self.xds_port = xds_host, xds_port - - @property - def xds_address(self) -> str: - if not self.xds_host: - return "" - if not self.xds_port: - return self.xds_host - return f"{self.xds_host}:{self.xds_port}" - - @property - def xds_uri(self) -> str: - if not self.xds_host: - return "" - return f"xds:///{self.xds_address}" - - def get_test_server(self) -> grpc_channelz.Server: - """Return channelz representation of a server running TestService. - - Raises: - GrpcApp.NotFound: Test server not found. - """ - server = self.channelz.find_server_listening_on_port(self.rpc_port) - if not server: - raise self.NotFound( - f"[{self.hostname}] Server" - f"listening on port {self.rpc_port} not found" - ) - return server - - def get_test_server_sockets(self) -> Iterator[grpc_channelz.Socket]: - """List all sockets of the test server. - - Raises: - GrpcApp.NotFound: Test server not found. - """ - server = self.get_test_server() - return self.channelz.list_server_sockets(server) - - def get_server_socket_matching_client( - self, client_socket: grpc_channelz.Socket - ): - """Find test server socket that matches given test client socket. - - Sockets are matched using TCP endpoints (ip:port), further on "address". - Server socket remote address matched with client socket local address. - - Raises: - GrpcApp.NotFound: Server socket matching client socket not found. - """ - client_local = self.channelz.sock_address_to_str(client_socket.local) - logger.debug( - "[%s] Looking for a server socket connected to the client %s", - self.hostname, - client_local, - ) - - server_socket = self.channelz.find_server_socket_matching_client( - self.get_test_server_sockets(), client_socket - ) - if not server_socket: - raise self.NotFound( - f"[{self.hostname}] Socket to client {client_local} not found" - ) - - logger.info( - "[%s] Found matching socket pair: server(%s) <-> client(%s)", - self.hostname, - self.channelz.sock_addresses_pretty(server_socket), - self.channelz.sock_addresses_pretty(client_socket), - ) - return server_socket diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py deleted file mode 100644 index d2990323223..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/base_testcase.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Base test case used for xds test suites.""" - -from typing import Optional -import unittest - -from absl import logging -from absl.testing import absltest - - -class BaseTestCase(absltest.TestCase): - def run(self, result: Optional[unittest.TestResult] = None) -> None: - super().run(result) - test_errors = [error for test, error in result.errors if test is self] - test_failures = [ - failure for test, failure in result.failures if test is self - ] - test_unexpected_successes = [ - test for test in result.unexpectedSuccesses if test is self - ] - test_skipped = next( - (reason for test, reason in result.skipped if test is self), - None, - ) - # Assume one test case will only have one status. - if test_errors or test_failures: - logging.info("----- TestCase %s FAILED -----", self.id()) - if test_errors: - self._print_error_list(test_errors, is_unexpected_error=True) - if test_failures: - self._print_error_list(test_failures) - elif test_unexpected_successes: - logging.info( - "----- TestCase %s UNEXPECTEDLY SUCCEEDED -----", self.id() - ) - elif test_skipped: - logging.info("----- TestCase %s SKIPPED -----", self.id()) - logging.info("Reason for skipping: %s", test_skipped) - else: - logging.info("----- TestCase %s PASSED -----", self.id()) - - def _print_error_list( - self, errors: list[str], is_unexpected_error: bool = False - ) -> None: - # FAILUREs are those errors explicitly signalled using - # the TestCase.assert*() methods. - for err in errors: - logging.error( - "%s Traceback in %s:\n%s", - "ERROR" if is_unexpected_error else "FAILURE", - self.id(), - err, - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/session_affinity_util.py b/tools/run_tests/xds_k8s_test_driver/framework/test_cases/session_affinity_util.py deleted file mode 100644 index 677efed5751..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/test_cases/session_affinity_util.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for stateful session affinity tests. - -These utilities must be shared between test environments that configure SSA -via Kubernetes CRDs and environments that configure SSA directly through the -networkservices.googleapis.com API. -""" - -import datetime -import logging -from typing import Sequence, Tuple - -from framework import xds_k8s_testcase -from framework.helpers import retryers - -_XdsKubernetesBaseTestCase = xds_k8s_testcase.XdsKubernetesBaseTestCase -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - -_SET_COOKIE_MAX_WAIT_SEC = 300 - - -def get_setcookie_headers( - metadatas_by_peer: dict[str, "MetadataByPeer"] -) -> dict[str, str]: - cookies = dict() - for peer, metadatas in metadatas_by_peer.items(): - for rpc_metadatas in metadatas.rpc_metadata: - for metadata in rpc_metadatas.metadata: - if metadata.key.lower() == "set-cookie": - cookies[peer] = metadata.value - return cookies - - -def assert_eventually_retrieve_cookie_and_server( - test: _XdsKubernetesBaseTestCase, - test_client: _XdsTestClient, - servers: Sequence[_XdsTestServer], -) -> Tuple[str, _XdsTestServer]: - """Retrieves the initial cookie and corresponding server. - - Given a test client and set of backends for which SSA is enabled, samples - a single RPC from the test client to the backends, with metadata collection enabled. - The "set-cookie" header is retrieved and its contents are returned along with the - server to which it corresponds. - - Since SSA config is supplied as a separate resource from the Route resource, - there will be periods of time where the SSA config may not be applied. This is - therefore an eventually consistent function. - """ - - def _assert_retrieve_cookie_and_server(): - lb_stats = test.assertSuccessfulRpcs(test_client, 1) - cookies = get_setcookie_headers(lb_stats.metadatas_by_peer) - test.assertLen(cookies, 1) - hostname = next(iter(cookies.keys())) - cookie = cookies[hostname] - - chosen_server_candidates = tuple( - srv for srv in servers if srv.hostname == hostname - ) - test.assertLen(chosen_server_candidates, 1) - chosen_server = chosen_server_candidates[0] - return cookie, chosen_server - - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta(seconds=10), - timeout=datetime.timedelta(seconds=_SET_COOKIE_MAX_WAIT_SEC), - log_level=logging.INFO, - ) - try: - return retryer(_assert_retrieve_cookie_and_server) - except retryers.RetryError as retry_error: - logging.exception( - "Rpcs did not go to expected servers before timeout %s", - _SET_COOKIE_MAX_WAIT_SEC, - ) - raise retry_error diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py deleted file mode 100644 index b710db04431..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_flags.py +++ /dev/null @@ -1,212 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket - -from absl import flags - -from framework.helpers import highlighter - -# GCP -PROJECT = flags.DEFINE_string( - "project", default=None, help="(required) GCP Project ID." -) -RESOURCE_PREFIX = flags.DEFINE_string( - "resource_prefix", - default=None, - help=( - "(required) The prefix used to name GCP resources.\n" - "Together with `resource_suffix` used to create unique " - "resource names." - ), -) -RESOURCE_SUFFIX = flags.DEFINE_string( - "resource_suffix", - default=None, - help=( - "The suffix used to name GCP resources.\n" - "Together with `resource_prefix` used to create unique " - "resource names.\n" - "(default: test suite will generate a random suffix, based on suite " - "resource management preferences)" - ), -) -NETWORK = flags.DEFINE_string( - "network", default="default", help="GCP Network ID" -) -COMPUTE_API_VERSION = flags.DEFINE_string( - "compute_api_version", - default="v1", - help="The version of the GCP Compute API, e.g., v1, v1alpha", -) -# Mirrors --xds-server-uri argument of Traffic Director gRPC Bootstrap -XDS_SERVER_URI = flags.DEFINE_string( - "xds_server_uri", default=None, help="Override Traffic Director server URI." -) -ENSURE_FIREWALL = flags.DEFINE_bool( - "ensure_firewall", - default=False, - help="Ensure the allow-health-check firewall exists before each test case", -) -FIREWALL_SOURCE_RANGE = flags.DEFINE_list( - "firewall_source_range", - default=["35.191.0.0/16", "130.211.0.0/22"], - help="Update the source range of the firewall rule.", -) -FIREWALL_ALLOWED_PORTS = flags.DEFINE_list( - "firewall_allowed_ports", - default=["8080-8100"], - help="Update the allowed ports of the firewall rule.", -) - -# Test server -SERVER_NAME = flags.DEFINE_string( - "server_name", - default="psm-grpc-server", - help="The name to use for test server deployments.", -) -SERVER_PORT = flags.DEFINE_integer( - "server_port", - default=8080, - lower_bound=1, - upper_bound=65535, - help="Server test port.\nMust be within --firewall_allowed_ports.", -) -SERVER_MAINTENANCE_PORT = flags.DEFINE_integer( - "server_maintenance_port", - default=None, - lower_bound=1, - upper_bound=65535, - help=( - "Server port running maintenance services: Channelz, CSDS, Health, " - "XdsUpdateHealth, and ProtoReflection (optional).\n" - "Must be within --firewall_allowed_ports.\n" - "(default: the port is chosen automatically based on " - "the security configuration)" - ), -) -SERVER_XDS_HOST = flags.DEFINE_string( - "server_xds_host", - default="xds-test-server", - help=( - "The xDS hostname of the test server.\n" - "Together with `server_xds_port` makes test server target URI, " - "xds:///hostname:port" - ), -) -# Note: port 0 known to represent a request for dynamically-allocated port -# https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Well-known_ports -SERVER_XDS_PORT = flags.DEFINE_integer( - "server_xds_port", - default=8080, - lower_bound=0, - upper_bound=65535, - help=( - "The xDS port of the test server.\n" - "Together with `server_xds_host` makes test server target URI, " - "xds:///hostname:port\n" - "Must be unique within a GCP project.\n" - "Set to 0 to select any unused port." - ), -) - -# Test client -CLIENT_NAME = flags.DEFINE_string( - "client_name", - default="psm-grpc-client", - help="The name to use for test client deployments", -) -CLIENT_PORT = flags.DEFINE_integer( - "client_port", - default=8079, - lower_bound=1, - upper_bound=65535, - help=( - "The port test client uses to run gRPC services: Channelz, CSDS, " - "XdsStats, XdsUpdateClientConfigure, and ProtoReflection (optional).\n" - "Doesn't have to be within --firewall_allowed_ports." - ), -) - -# Testing metadata -TESTING_VERSION = flags.DEFINE_string( - "testing_version", - default=None, - help="The testing gRPC version branch name. Like master, dev, v1.55.x", -) - -FORCE_CLEANUP = flags.DEFINE_bool( - "force_cleanup", - default=False, - help="Force resource cleanup, even if not created by this test run", -) - -COLLECT_APP_LOGS = flags.DEFINE_bool( - "collect_app_logs", - default=False, - help=( - f"Collect the logs of the xDS Test Client and Server\n" - f"into the test_app_logs/ directory under the log directory.\n" - f"See --log_dir description for configuring the log directory." - ), -) - -# Needed to configure urllib3 socket timeout, which is infinity by default. -SOCKET_DEFAULT_TIMEOUT = flags.DEFINE_float( - "socket_default_timeout", - default=60, - lower_bound=0, - help=( - "Set the default timeout in seconds on blocking socket operations.\n" - "If zero is given, the new sockets have no timeout. " - ), -) - -GFE_DEBUG_HEADER = flags.DEFINE_enum( - "gfe_debug_header", - default=None, - enum_values=[ - "gfe_response_only", - "all_response", - "request_and_response", - ], - help="Whether to enable GFE debug headers and what value to use.", -) - - -def set_socket_default_timeout_from_flag() -> None: - """A helper to configure default socket timeout from a flag. - - This is known to affect the following pip packages: - - google-api-python-client: has the default timeout set to 60: - https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http-module.html#build_http - - kubernetes: falls back to urllib3 timeout, which is infinity by default: - https://urllib3.readthedocs.io/en/stable/reference/urllib3.util.html#urllib3.util.Timeout - - NOTE: Must be called _after_ the flags were parsed by absl, but before - the before KubernetesApiManager or GcpApiManager initialized. - """ - timeout: float = SOCKET_DEFAULT_TIMEOUT.value - # None is inf timeout, which is represented by 0 in the flag. - socket.setdefaulttimeout(None if timeout == 0 else timeout) - - -flags.adopt_module_key_flags(highlighter) - -flags.mark_flags_as_required( - [ - "project", - "resource_prefix", - ] -) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_gamma_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_gamma_testcase.py deleted file mode 100644 index 0d86a15bb76..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_gamma_testcase.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime -import logging -from typing import Optional - -from framework.infrastructure import k8s -import framework.infrastructure.traffic_director_gamma as td_gamma -from framework.test_app import client_app -from framework.test_app import server_app -from framework.test_app.runners.k8s import gamma_server_runner -from framework.test_app.runners.k8s import k8s_xds_client_runner -import framework.xds_k8s_testcase as xds_k8s_testcase - -GammaServerRunner = gamma_server_runner.GammaServerRunner -KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -XdsTestClient = client_app.XdsTestClient -XdsTestServer = server_app.XdsTestServer - -logger = logging.getLogger(__name__) - - -# TODO(sergiitk): [GAMMA] Move into framework/test_cases -class GammaXdsKubernetesTestCase(xds_k8s_testcase.RegularXdsKubernetesTestCase): - server_runner: GammaServerRunner - frontend_service_name: str - pre_stop_hook: Optional[bool] = None - termination_grace_period_seconds: int = 0 - - def setUp(self): - """Hook method for setting up the test fixture before exercising it.""" - # TODO(sergiitk): [GAMMA] Remove when refactored to be TD-manager-less. - # pylint: disable=bad-super-call - # Skips RegularXdsKubernetesTestCase and IsolatedXdsKubernetesTestCase - # and calls setUp on XdsKubernetesBaseTestCase. - # IsolatedXdsKubernetesTestCase randomizes server_xds_port when it's 0, - # and in GAMMA we always need it unset. - # Calls XdsKubernetesBaseTestCase.setUp(): - super(xds_k8s_testcase.IsolatedXdsKubernetesTestCase, self).setUp() - # pylint: enable=bad-super-call - - if self.pre_stop_hook is None: - self.pre_stop_hook = False - - # Random suffix per test. - self.createRandomSuffix() - - # TODO(sergiitk): [GAMMA] Make a TD-manager-less base test case - # TD Manager - self.td = self.initTrafficDirectorManager() - - # Generate unique mesh name too. - self.frontend_service_name = ( - f"{self.resource_prefix}-{self.resource_suffix.lower()}" - ) - - # Test Server runner - self.server_namespace = GammaServerRunner.make_namespace_name( - self.resource_prefix, self.resource_suffix - ) - self.server_runner = self.initKubernetesServerRunner() - - # Test Client runner - self.client_namespace = KubernetesClientRunner.make_namespace_name( - self.resource_prefix, self.resource_suffix - ) - self.client_runner = self.initKubernetesClientRunner() - - # Cleanup. - self.force_cleanup = True - self.force_cleanup_namespace = True - - # TODO(sergiitk): [GAMMA] Make a TD-manager-less base test case - def initTrafficDirectorManager( - self, - ) -> td_gamma.TrafficDirectorGammaManager: - return td_gamma.TrafficDirectorGammaManager( - self.gcp_api_manager, - project=self.project, - resource_prefix=self.resource_prefix, - resource_suffix=self.resource_suffix, - network=self.network, - compute_api_version=self.compute_api_version, - ) - - def initKubernetesServerRunner(self) -> GammaServerRunner: - return GammaServerRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.server_namespace - ), - self.frontend_service_name, - deployment_name=self.server_name, - image_name=self.server_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - termination_grace_period_seconds=self.termination_grace_period_seconds, - pre_stop_hook=self.pre_stop_hook, - ) - - def startTestClient( - self, test_server: XdsTestServer, **kwargs - ) -> XdsTestClient: - server_target = ( - f"xds:///{self.frontend_service_name}" - f".{self.server_namespace}.svc.cluster.local" - f":{test_server.rpc_port}" - ) - kwargs.setdefault("generate_mesh_id", True) - # Waiting for an active channel takes less time in non-gamma - # test suites because they only start waiting after already waited for - # the TD backends to be created and report healthy. - # In GAMMA, these resources are created asynchronously by Kubernetes. - # To compensate for this, we double the timeout for GAMMA tests. - return self._start_test_client( - server_target, - wait_for_server_channel_ready_timeout=datetime.timedelta( - minutes=10 - ), - **kwargs, - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_flags.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_flags.py deleted file mode 100644 index dc415e9fd4d..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_flags.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from absl import flags - -# GCP -KUBE_CONTEXT = flags.DEFINE_string( - "kube_context", default=None, help="Kubectl context to use" -) -SECONDARY_KUBE_CONTEXT = flags.DEFINE_string( - "secondary_kube_context", - default=None, - help="Secondary kubectl context to use for cluster in another region", -) -GCP_SERVICE_ACCOUNT = flags.DEFINE_string( - "gcp_service_account", - default=None, - help="GCP Service account for GKE workloads to impersonate", -) -TD_BOOTSTRAP_IMAGE = flags.DEFINE_string( - "td_bootstrap_image", - default=None, - help="Traffic Director gRPC Bootstrap Docker image", -) - -# Test app -SERVER_IMAGE = flags.DEFINE_string( - "server_image", default=None, help="Server Docker image name" -) -SERVER_IMAGE_CANONICAL = flags.DEFINE_string( - "server_image_canonical", - default=None, - help=( - "The canonical implementation of the xDS test server.\n" - "Can be used in tests where language-specific xDS test server" - "does not exist, or missing a feature required for the test." - ), -) -CLIENT_IMAGE = flags.DEFINE_string( - "client_image", default=None, help="Client Docker image name" -) -DEBUG_USE_PORT_FORWARDING = flags.DEFINE_bool( - "debug_use_port_forwarding", - default=False, - help="Development only: use kubectl port-forward to connect to test app", -) -ENABLE_WORKLOAD_IDENTITY = flags.DEFINE_bool( - "enable_workload_identity", - default=True, - help="Enable the WorkloadIdentity feature", -) - -flags.mark_flags_as_required( - [ - "kube_context", - "td_bootstrap_image", - "server_image", - "client_image", - ] -) - - -def require_secondary_context(filename: str): - """ - Makes secondary_kube_context flag required and adds the non-empty validator. - - Typical usage example: - - xds_k8s_flags.require_secondary_context(__file__) - """ - flags.mark_flag_as_required("secondary_kube_context") - - def _val_not_empty(val: str) -> bool: - # Do not allow whitespace-only values to produce a better error. - return bool(val.strip()) - - flags.register_validator( - "secondary_kube_context", - _val_not_empty, - message=( - f"{filename} requires non-empty secondary_kube_context to access" - " the secondary k8s cluster" - ), - ) diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py deleted file mode 100644 index 55743de13b2..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py +++ /dev/null @@ -1,1212 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import abc -import contextlib -import datetime -import enum -import hashlib -import logging -import re -import signal -import time -from types import FrameType -from typing import Any, Callable, List, Optional, Tuple, Union - -from absl import flags -from absl.testing import absltest -from google.protobuf import json_format -import grpc - -from framework import xds_flags -from framework import xds_k8s_flags -from framework import xds_url_map_testcase -from framework.helpers import grpc as helpers_grpc -from framework.helpers import rand as helpers_rand -from framework.helpers import retryers -from framework.helpers import skips -import framework.helpers.highlighter -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.infrastructure import traffic_director -from framework.rpc import grpc_channelz -from framework.rpc import grpc_csds -from framework.rpc import grpc_testing -from framework.test_app import client_app -from framework.test_app import server_app -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner -from framework.test_cases import base_testcase - -logger = logging.getLogger(__name__) -# TODO(yashkt): We will no longer need this flag once Core exposes local certs -# from channelz -_CHECK_LOCAL_CERTS = flags.DEFINE_bool( - "check_local_certs", - default=True, - help="Security Tests also check the value of local certs", -) -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) - -# Type aliases -TrafficDirectorManager = traffic_director.TrafficDirectorManager -TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager -TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager -XdsTestServer = server_app.XdsTestServer -XdsTestClient = client_app.XdsTestClient -KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner -KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -_LoadBalancerStatsResponse = grpc_testing.LoadBalancerStatsResponse -_LoadBalancerAccumulatedStatsResponse = ( - grpc_testing.LoadBalancerAccumulatedStatsResponse -) -_ChannelState = grpc_channelz.ChannelState -_timedelta = datetime.timedelta -ClientConfig = grpc_csds.ClientConfig -RpcMetadata = grpc_testing.LoadBalancerStatsResponse.RpcMetadata -MetadataByPeer: list[str, RpcMetadata] -# pylint complains about signal.Signals for some reason. -_SignalNum = Union[int, signal.Signals] # pylint: disable=no-member -_SignalHandler = Callable[[_SignalNum, Optional[FrameType]], Any] - -_TD_CONFIG_MAX_WAIT_SEC = 600 - - -def evaluate_test_config( - check: Callable[[skips.TestConfig], bool] -) -> skips.TestConfig: - """Evaluates the test config check against Abseil flags. - - TODO(sergiitk): split into parse_lang_spec and check_is_supported. - """ - # NOTE(lidiz) a manual skip mechanism is needed because absl/flags - # cannot be used in the built-in test-skipping decorators. See the - # official FAQs: - # https://abseil.io/docs/python/guides/flags#faqs - test_config = skips.TestConfig( - client_lang=skips.get_lang(xds_k8s_flags.CLIENT_IMAGE.value), - server_lang=skips.get_lang(xds_k8s_flags.SERVER_IMAGE.value), - version=xds_flags.TESTING_VERSION.value, - ) - if not check(test_config): - logger.info("Skipping %s", test_config) - raise absltest.SkipTest(f"Unsupported test config: {test_config}") - - logger.info("Detected language and version: %s", test_config) - return test_config - - -class TdPropagationRetryableError(Exception): - """Indicates that TD config hasn't propagated yet, and it's safe to retry""" - - -class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase): - lang_spec: skips.TestConfig - client_namespace: str - client_runner: KubernetesClientRunner - ensure_firewall: bool - force_cleanup: bool - gcp_api_manager: gcp.api.GcpApiManager - gcp_service_account: Optional[str] - k8s_api_manager: k8s.KubernetesApiManager - secondary_k8s_api_manager: Optional[k8s.KubernetesApiManager] = None - network: str - project: str - resource_prefix: str - resource_suffix: str = "" - # Whether to randomize resources names for each test by appending a - # unique suffix. - resource_suffix_randomize: bool = True - server_maintenance_port: Optional[int] - server_namespace: str - server_runner: KubernetesServerRunner - server_xds_host: str - server_xds_port: Optional[int] - td: TrafficDirectorManager - td_bootstrap_image: str - _prev_sigint_handler: Optional[_SignalHandler] = None - _handling_sigint: bool = False - yaml_highlighter: framework.helpers.highlighter.HighlighterYaml = None - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - """Overridden by the test class to decide if the config is supported. - - Returns: - A bool indicates if the given config is supported. - """ - del config - return True - - @classmethod - def setUpClass(cls): - """Hook method for setting up class fixture before running tests in - the class. - """ - logger.info("----- Testing %s -----", cls.__name__) - logger.info("Logs timezone: %s", time.localtime().tm_zone) - - # Raises unittest.SkipTest if given client/server/version does not - # support current test case. - cls.lang_spec = evaluate_test_config(cls.is_supported) - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - # GCP - cls.project = xds_flags.PROJECT.value - cls.network = xds_flags.NETWORK.value - cls.gcp_service_account = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value - cls.td_bootstrap_image = xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value - cls.xds_server_uri = xds_flags.XDS_SERVER_URI.value - cls.ensure_firewall = xds_flags.ENSURE_FIREWALL.value - cls.firewall_allowed_ports = xds_flags.FIREWALL_ALLOWED_PORTS.value - cls.compute_api_version = xds_flags.COMPUTE_API_VERSION.value - - # Resource names. - cls.resource_prefix = xds_flags.RESOURCE_PREFIX.value - if xds_flags.RESOURCE_SUFFIX.value is not None: - cls.resource_suffix_randomize = False - cls.resource_suffix = xds_flags.RESOURCE_SUFFIX.value - - # Test server - cls.server_image = xds_k8s_flags.SERVER_IMAGE.value - cls.server_name = xds_flags.SERVER_NAME.value - cls.server_port = xds_flags.SERVER_PORT.value - cls.server_maintenance_port = xds_flags.SERVER_MAINTENANCE_PORT.value - cls.server_xds_host = xds_flags.SERVER_NAME.value - cls.server_xds_port = xds_flags.SERVER_XDS_PORT.value - - # Test client - cls.client_image = xds_k8s_flags.CLIENT_IMAGE.value - cls.client_name = xds_flags.CLIENT_NAME.value - cls.client_port = xds_flags.CLIENT_PORT.value - - # Test suite settings - cls.force_cleanup = xds_flags.FORCE_CLEANUP.value - cls.force_cleanup_namespace = xds_flags.FORCE_CLEANUP.value - cls.debug_use_port_forwarding = ( - xds_k8s_flags.DEBUG_USE_PORT_FORWARDING.value - ) - cls.enable_workload_identity = ( - xds_k8s_flags.ENABLE_WORKLOAD_IDENTITY.value - ) - cls.check_local_certs = _CHECK_LOCAL_CERTS.value - - # Resource managers - cls.k8s_api_manager = k8s.KubernetesApiManager( - xds_k8s_flags.KUBE_CONTEXT.value - ) - if xds_k8s_flags.SECONDARY_KUBE_CONTEXT.value is not None: - cls.secondary_k8s_api_manager = k8s.KubernetesApiManager( - xds_k8s_flags.SECONDARY_KUBE_CONTEXT.value - ) - cls.gcp_api_manager = gcp.api.GcpApiManager() - - # Other - cls.yaml_highlighter = framework.helpers.highlighter.HighlighterYaml() - - @classmethod - def _pretty_accumulated_stats( - cls, - accumulated_stats: _LoadBalancerAccumulatedStatsResponse, - *, - ignore_empty: bool = False, - highlight: bool = True, - ) -> str: - stats_yaml = helpers_grpc.accumulated_stats_pretty( - accumulated_stats, ignore_empty=ignore_empty - ) - if not highlight: - return stats_yaml - return cls.yaml_highlighter.highlight(stats_yaml) - - @classmethod - def _pretty_lb_stats(cls, lb_stats: _LoadBalancerStatsResponse) -> str: - stats_yaml = helpers_grpc.lb_stats_pretty(lb_stats) - return cls.yaml_highlighter.highlight(stats_yaml) - - @classmethod - def tearDownClass(cls): - cls.k8s_api_manager.close() - if cls.secondary_k8s_api_manager is not None: - cls.secondary_k8s_api_manager.close() - cls.gcp_api_manager.close() - - def setUp(self): - self._prev_sigint_handler = signal.signal( - signal.SIGINT, self.handle_sigint - ) - - def handle_sigint( - self, signalnum: _SignalNum, frame: Optional[FrameType] - ) -> None: - logger.info("Caught Ctrl+C, cleaning up...") - self._handling_sigint = True - # Force resource cleanup by their name. Addresses the case where ctrl-c - # is pressed while waiting for the resource creation. - self.force_cleanup = True - self.tearDown() - self.tearDownClass() - self._handling_sigint = False - if self._prev_sigint_handler is not None: - signal.signal(signal.SIGINT, self._prev_sigint_handler) - raise KeyboardInterrupt - - @contextlib.contextmanager - def subTest(self, msg, **params): # noqa pylint: disable=signature-differs - logger.info("--- Starting subTest %s.%s ---", self.id(), msg) - try: - yield super().subTest(msg, **params) - finally: - if not self._handling_sigint: - logger.info("--- Finished subTest %s.%s ---", self.id(), msg) - - def setupTrafficDirectorGrpc(self): - self.td.setup_for_grpc( - self.server_xds_host, - self.server_xds_port, - health_check_port=self.server_maintenance_port, - ) - - def setupServerBackends( - self, - *, - wait_for_healthy_status=True, - server_runner=None, - max_rate_per_endpoint: Optional[int] = None, - ): - if server_runner is None: - server_runner = self.server_runner - # Load Backends - ( - neg_name, - neg_zones, - ) = server_runner.k8s_namespace.parse_service_neg_status( - server_runner.service_name, self.server_port - ) - - # Add backends to the Backend Service - self.td.backend_service_add_neg_backends( - neg_name, neg_zones, max_rate_per_endpoint=max_rate_per_endpoint - ) - if wait_for_healthy_status: - self.td.wait_for_backends_healthy_status() - - def removeServerBackends(self, *, server_runner=None): - if server_runner is None: - server_runner = self.server_runner - # Load Backends - ( - neg_name, - neg_zones, - ) = server_runner.k8s_namespace.parse_service_neg_status( - server_runner.service_name, self.server_port - ) - - # Remove backends from the Backend Service - self.td.backend_service_remove_neg_backends(neg_name, neg_zones) - - def assertSuccessfulRpcs( - self, test_client: XdsTestClient, num_rpcs: int = 100 - ) -> _LoadBalancerStatsResponse: - lb_stats = self.getClientRpcStats(test_client, num_rpcs) - self.assertAllBackendsReceivedRpcs(lb_stats) - failed = int(lb_stats.num_failures) - self.assertLessEqual( - failed, - 0, - msg=f"Expected all RPCs to succeed: {failed} of {num_rpcs} failed", - ) - return lb_stats - - @staticmethod - def diffAccumulatedStatsPerMethod( - before: _LoadBalancerAccumulatedStatsResponse, - after: _LoadBalancerAccumulatedStatsResponse, - ) -> _LoadBalancerAccumulatedStatsResponse: - """Only diffs stats_per_method, as the other fields are deprecated.""" - diff = _LoadBalancerAccumulatedStatsResponse() - for method, method_stats in after.stats_per_method.items(): - for status, count in method_stats.result.items(): - count -= before.stats_per_method[method].result[status] - if count < 0: - raise AssertionError("Diff of count shouldn't be negative") - if count > 0: - diff.stats_per_method[method].result[status] = count - rpcs_started = ( - method_stats.rpcs_started - - before.stats_per_method[method].rpcs_started - ) - if rpcs_started < 0: - raise AssertionError("Diff of count shouldn't be negative") - diff.stats_per_method[method].rpcs_started = rpcs_started - return diff - - def assertRpcStatusCodes( - self, - test_client: XdsTestClient, - *, - expected_status: grpc.StatusCode, - duration: _timedelta, - method: str, - stray_rpc_limit: int = 0, - ) -> None: - """Assert all RPCs for a method are completing with a certain status.""" - # pylint: disable=too-many-locals - expected_status_int: int = expected_status.value[0] - expected_status_fmt: str = helpers_grpc.status_pretty(expected_status) - - # Sending with pre-set QPS for a period of time - before_stats = test_client.get_load_balancer_accumulated_stats() - logging.debug( - ( - "[%s] << LoadBalancerAccumulatedStatsResponse initial" - " measurement:\n%s" - ), - test_client.hostname, - self._pretty_accumulated_stats(before_stats), - ) - - time.sleep(duration.total_seconds()) - - after_stats = test_client.get_load_balancer_accumulated_stats() - logging.debug( - ( - "[%s] << LoadBalancerAccumulatedStatsResponse after %s seconds:" - "\n%s" - ), - test_client.hostname, - duration.total_seconds(), - self._pretty_accumulated_stats(after_stats), - ) - - diff_stats = self.diffAccumulatedStatsPerMethod( - before_stats, after_stats - ) - logger.info( - ( - "[%s] << Received accumulated stats difference." - " Expecting RPCs with status %s for method %s:\n%s" - ), - test_client.hostname, - expected_status_fmt, - method, - self._pretty_accumulated_stats(diff_stats, ignore_empty=True), - ) - - # Used in stack traces. Don't highlight for better compatibility. - diff_stats_fmt: str = self._pretty_accumulated_stats( - diff_stats, ignore_empty=True, highlight=False - ) - - # 1. Verify the completed RPCs of the given method has no statuses - # other than the expected_status, - stats = diff_stats.stats_per_method[method] - for found_status_int, count in stats.result.items(): - found_status = helpers_grpc.status_from_int(found_status_int) - if found_status != expected_status and count > stray_rpc_limit: - self.fail( - f"Expected only status {expected_status_fmt}," - " but found status" - f" {helpers_grpc.status_pretty(found_status)}" - f" for method {method}." - f"\nDiff stats:\n{diff_stats_fmt}" - ) - - # 2. Verify there are completed RPCs of the given method with - # the expected_status. - self.assertGreater( - stats.result[expected_status_int], - 0, - msg=( - "Expected non-zero completed RPCs with status" - f" {expected_status_fmt} for method {method}." - f"\nDiff stats:\n{diff_stats_fmt}" - ), - ) - - def assertRpcsEventuallyGoToGivenServers( - self, - test_client: XdsTestClient, - servers: List[XdsTestServer], - num_rpcs: int = 100, - ): - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta(seconds=1), - timeout=datetime.timedelta(seconds=_TD_CONFIG_MAX_WAIT_SEC), - log_level=logging.INFO, - ) - try: - retryer( - self._assertRpcsEventuallyGoToGivenServers, - test_client, - servers, - num_rpcs, - ) - except retryers.RetryError as retry_error: - logger.exception( - "Rpcs did not go to expected servers before timeout %s", - _TD_CONFIG_MAX_WAIT_SEC, - ) - raise retry_error - - def _assertRpcsEventuallyGoToGivenServers( - self, - test_client: XdsTestClient, - servers: List[XdsTestServer], - num_rpcs: int, - ): - server_hostnames = [server.hostname for server in servers] - logger.info("Verifying RPCs go to servers %s", server_hostnames) - lb_stats = self.getClientRpcStats(test_client, num_rpcs) - failed = int(lb_stats.num_failures) - self.assertLessEqual( - failed, - 0, - msg=f"Expected all RPCs to succeed: {failed} of {num_rpcs} failed", - ) - for server_hostname in server_hostnames: - self.assertIn( - server_hostname, - lb_stats.rpcs_by_peer, - f"Server {server_hostname} did not receive RPCs", - ) - for server_hostname in lb_stats.rpcs_by_peer.keys(): - self.assertIn( - server_hostname, - server_hostnames, - f"Unexpected server {server_hostname} received RPCs", - ) - - def assertXdsConfigExists(self, test_client: XdsTestClient): - config = test_client.csds.fetch_client_status(log_level=logging.INFO) - self.assertIsNotNone(config) - seen = set() - want = frozenset( - [ - "listener_config", - "cluster_config", - "route_config", - "endpoint_config", - ] - ) - for xds_config in config.xds_config: - seen.add(xds_config.WhichOneof("per_xds_config")) - for generic_xds_config in config.generic_xds_configs: - if re.search(r"\.Listener$", generic_xds_config.type_url): - seen.add("listener_config") - elif re.search( - r"\.RouteConfiguration$", generic_xds_config.type_url - ): - seen.add("route_config") - elif re.search(r"\.Cluster$", generic_xds_config.type_url): - seen.add("cluster_config") - elif re.search( - r"\.ClusterLoadAssignment$", generic_xds_config.type_url - ): - seen.add("endpoint_config") - logger.debug( - "Received xDS config dump: %s", - json_format.MessageToJson(config, indent=2), - ) - self.assertSameElements(want, seen) - - def assertRouteConfigUpdateTrafficHandoff( - self, - test_client: XdsTestClient, - previous_route_config_version: str, - retry_wait_second: int, - timeout_second: int, - ): - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta(seconds=retry_wait_second), - timeout=datetime.timedelta(seconds=timeout_second), - retry_on_exceptions=(TdPropagationRetryableError,), - logger=logger, - log_level=logging.INFO, - ) - try: - for attempt in retryer: - with attempt: - self.assertSuccessfulRpcs(test_client) - raw_config = test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - dumped_config = xds_url_map_testcase.DumpedXdsConfig( - json_format.MessageToDict(raw_config) - ) - route_config_version = dumped_config.rds_version - if previous_route_config_version == route_config_version: - logger.info( - "Routing config not propagated yet. Retrying." - ) - raise TdPropagationRetryableError( - "CSDS not get updated routing config corresponding" - " to the second set of url maps" - ) - else: - self.assertSuccessfulRpcs(test_client) - logger.info( - ( - "[SUCCESS] Confirmed successful RPC with the " - "updated routing config, version=%s" - ), - route_config_version, - ) - except retryers.RetryError as retry_error: - logger.info( - ( - "Retry exhausted. TD routing config propagation failed" - " after timeout %ds. Last seen client config dump: %s" - ), - timeout_second, - dumped_config, - ) - raise retry_error - - def assertFailedRpcs( - self, test_client: XdsTestClient, num_rpcs: Optional[int] = 100 - ): - lb_stats = self.getClientRpcStats(test_client, num_rpcs) - failed = int(lb_stats.num_failures) - self.assertEqual( - failed, - num_rpcs, - msg=f"Expected all RPCs to fail: {failed} of {num_rpcs} failed", - ) - - def getClientRpcStats( - self, - test_client: XdsTestClient, - num_rpcs: int, - *, - metadata_keys: Optional[tuple[str, ...]] = None, - ) -> _LoadBalancerStatsResponse: - lb_stats = test_client.get_load_balancer_stats( - num_rpcs=num_rpcs, - metadata_keys=metadata_keys, - ) - logger.info( - "[%s] << Received LoadBalancerStatsResponse:\n%s", - test_client.hostname, - self._pretty_lb_stats(lb_stats), - ) - return lb_stats - - def assertAllBackendsReceivedRpcs(self, lb_stats): - # TODO(sergiitk): assert backends length - for backend, rpcs_count in lb_stats.rpcs_by_peer.items(): - self.assertGreater( - int(rpcs_count), - 0, - msg=f"Backend {backend} did not receive a single RPC", - ) - - -class IsolatedXdsKubernetesTestCase( - XdsKubernetesBaseTestCase, metaclass=abc.ABCMeta -): - """Isolated test case. - - Base class for tests cases where infra resources are created before - each test, and destroyed after. - """ - - def setUp(self): - """Hook method for setting up the test fixture before exercising it.""" - super().setUp() - - # Random suffix per test. - self.createRandomSuffix() - - # TD Manager - self.td = self.initTrafficDirectorManager() - - # Test Server runner - self.server_namespace = KubernetesServerRunner.make_namespace_name( - self.resource_prefix, self.resource_suffix - ) - self.server_runner = self.initKubernetesServerRunner() - - # Test Client runner - self.client_namespace = KubernetesClientRunner.make_namespace_name( - self.resource_prefix, self.resource_suffix - ) - self.client_runner = self.initKubernetesClientRunner() - - # Ensures the firewall exist - if self.ensure_firewall: - self.td.create_firewall_rule( - allowed_ports=self.firewall_allowed_ports - ) - - # Randomize xds port, when it's set to 0 - if self.server_xds_port == 0: - # TODO(sergiitk): this is prone to race conditions: - # The port might not me taken now, but there's not guarantee - # it won't be taken until the tests get to creating - # forwarding rule. This check is better than nothing, - # but we should find a better approach. - self.server_xds_port = self.td.find_unused_forwarding_rule_port() - logger.info("Found unused xds port: %s", self.server_xds_port) - - def createRandomSuffix(self): - if self.resource_suffix_randomize: - self.resource_suffix = helpers_rand.random_resource_suffix() - logger.info( - "Test run resource prefix: %s, suffix: %s", - self.resource_prefix, - self.resource_suffix, - ) - - @abc.abstractmethod - def initTrafficDirectorManager(self) -> TrafficDirectorManager: - raise NotImplementedError - - @abc.abstractmethod - def initKubernetesServerRunner(self) -> KubernetesServerRunner: - raise NotImplementedError - - @abc.abstractmethod - def initKubernetesClientRunner(self) -> KubernetesClientRunner: - raise NotImplementedError - - def tearDown(self): - logger.info("----- TestMethod %s teardown -----", self.id()) - logger.debug("Getting pods restart times") - client_restarts: int = 0 - server_restarts: int = 0 - try: - client_restarts = self.client_runner.get_pod_restarts( - self.client_runner.deployment - ) - server_restarts = self.server_runner.get_pod_restarts( - self.server_runner.deployment - ) - except (retryers.RetryError, k8s.NotFound) as e: - logger.exception(e) - - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=10), - attempts=3, - log_level=logging.INFO, - ) - try: - retryer(self.cleanup) - except retryers.RetryError: - logger.exception("Got error during teardown") - finally: - logger.info("----- Test client/server logs -----") - self.client_runner.logs_explorer_run_history_links() - self.server_runner.logs_explorer_run_history_links() - - # Fail if any of the pods restarted. - self.assertEqual( - client_restarts, - 0, - msg=( - "Client container unexpectedly restarted" - f" {client_restarts} times during test. In most cases, this" - " is caused by the test client app crash." - ), - ) - self.assertEqual( - server_restarts, - 0, - msg=( - "Server container unexpectedly restarted" - f" {server_restarts} times during test. In most cases, this" - " is caused by the test client app crash." - ), - ) - - def cleanup(self): - self.td.cleanup(force=self.force_cleanup) - self.client_runner.cleanup(force=self.force_cleanup) - self.server_runner.cleanup( - force=self.force_cleanup, force_namespace=self.force_cleanup - ) - - def _start_test_client( - self, - server_target: str, - *, - wait_for_active_ads: bool = True, - wait_for_server_channel_ready: bool = True, - wait_for_active_ads_timeout: Optional[_timedelta] = None, - wait_for_server_channel_ready_timeout: Optional[_timedelta] = None, - **kwargs, - ) -> XdsTestClient: - test_client = self.client_runner.run( - server_target=server_target, **kwargs - ) - if wait_for_active_ads: - test_client.wait_for_active_xds_channel( - xds_server_uri=self.xds_server_uri, - timeout=wait_for_active_ads_timeout, - ) - if wait_for_server_channel_ready: - test_client.wait_for_server_channel_ready( - timeout=wait_for_server_channel_ready_timeout, - ) - return test_client - - -class RegularXdsKubernetesTestCase(IsolatedXdsKubernetesTestCase): - """Regular test case base class for testing PSM features in isolation.""" - - @classmethod - def setUpClass(cls): - """Hook method for setting up class fixture before running tests in - the class. - """ - super().setUpClass() - if cls.server_maintenance_port is None: - cls.server_maintenance_port = ( - KubernetesServerRunner.DEFAULT_MAINTENANCE_PORT - ) - - def initTrafficDirectorManager(self) -> TrafficDirectorManager: - return TrafficDirectorManager( - self.gcp_api_manager, - project=self.project, - resource_prefix=self.resource_prefix, - resource_suffix=self.resource_suffix, - network=self.network, - compute_api_version=self.compute_api_version, - ) - - def initKubernetesServerRunner(self) -> KubernetesServerRunner: - return KubernetesServerRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.server_namespace - ), - deployment_name=self.server_name, - image_name=self.server_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - ) - - def initKubernetesClientRunner(self) -> KubernetesClientRunner: - return KubernetesClientRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.client_namespace - ), - deployment_name=self.client_name, - image_name=self.client_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - stats_port=self.client_port, - reuse_namespace=self.server_namespace == self.client_namespace, - ) - - def startTestServers( - self, replica_count=1, server_runner=None, **kwargs - ) -> List[XdsTestServer]: - if server_runner is None: - server_runner = self.server_runner - test_servers = server_runner.run( - replica_count=replica_count, - test_port=self.server_port, - maintenance_port=self.server_maintenance_port, - **kwargs, - ) - for test_server in test_servers: - test_server.set_xds_address( - self.server_xds_host, self.server_xds_port - ) - return test_servers - - def startTestClient( - self, test_server: XdsTestServer, **kwargs - ) -> XdsTestClient: - return self._start_test_client(test_server.xds_uri, **kwargs) - - -class AppNetXdsKubernetesTestCase(RegularXdsKubernetesTestCase): - td: TrafficDirectorAppNetManager - - def initTrafficDirectorManager(self) -> TrafficDirectorAppNetManager: - return TrafficDirectorAppNetManager( - self.gcp_api_manager, - project=self.project, - resource_prefix=self.resource_prefix, - resource_suffix=self.resource_suffix, - network=self.network, - compute_api_version=self.compute_api_version, - ) - - -class SecurityXdsKubernetesTestCase(IsolatedXdsKubernetesTestCase): - """Test case base class for testing PSM security features in isolation.""" - - td: TrafficDirectorSecureManager - - class SecurityMode(enum.Enum): - MTLS = enum.auto() - TLS = enum.auto() - PLAINTEXT = enum.auto() - - @classmethod - def setUpClass(cls): - """Hook method for setting up class fixture before running tests in - the class. - """ - super().setUpClass() - if cls.server_maintenance_port is None: - # In secure mode, the maintenance port is different from - # the test port to keep it insecure, and make - # Health Checks and Channelz tests available. - # When not provided, use explicit numeric port value, so - # Backend Health Checks are created on a fixed port. - cls.server_maintenance_port = ( - KubernetesServerRunner.DEFAULT_SECURE_MODE_MAINTENANCE_PORT - ) - - def initTrafficDirectorManager(self) -> TrafficDirectorSecureManager: - return TrafficDirectorSecureManager( - self.gcp_api_manager, - project=self.project, - resource_prefix=self.resource_prefix, - resource_suffix=self.resource_suffix, - network=self.network, - compute_api_version=self.compute_api_version, - ) - - def initKubernetesServerRunner(self) -> KubernetesServerRunner: - return KubernetesServerRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.server_namespace - ), - deployment_name=self.server_name, - image_name=self.server_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - network=self.network, - xds_server_uri=self.xds_server_uri, - deployment_template="server-secure.deployment.yaml", - debug_use_port_forwarding=self.debug_use_port_forwarding, - ) - - def initKubernetesClientRunner(self) -> KubernetesClientRunner: - return KubernetesClientRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.client_namespace - ), - deployment_name=self.client_name, - image_name=self.client_image, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - xds_server_uri=self.xds_server_uri, - network=self.network, - deployment_template="client-secure.deployment.yaml", - stats_port=self.client_port, - reuse_namespace=self.server_namespace == self.client_namespace, - debug_use_port_forwarding=self.debug_use_port_forwarding, - ) - - def startSecureTestServer(self, replica_count=1, **kwargs) -> XdsTestServer: - test_server = self.server_runner.run( - replica_count=replica_count, - test_port=self.server_port, - maintenance_port=self.server_maintenance_port, - secure_mode=True, - **kwargs, - )[0] - test_server.set_xds_address(self.server_xds_host, self.server_xds_port) - return test_server - - def setupSecurityPolicies( - self, *, server_tls, server_mtls, client_tls, client_mtls - ): - self.td.setup_client_security( - server_namespace=self.server_namespace, - server_name=self.server_name, - tls=client_tls, - mtls=client_mtls, - ) - self.td.setup_server_security( - server_namespace=self.server_namespace, - server_name=self.server_name, - server_port=self.server_port, - tls=server_tls, - mtls=server_mtls, - ) - - def startSecureTestClient( - self, - test_server: XdsTestServer, - *, - wait_for_server_channel_ready=True, - **kwargs, - ) -> XdsTestClient: - return self._start_test_client( - server_target=test_server.xds_uri, - wait_for_server_channel_ready=wait_for_server_channel_ready, - secure_mode=True, - **kwargs, - ) - - def assertTestAppSecurity( - self, - mode: SecurityMode, - test_client: XdsTestClient, - test_server: XdsTestServer, - ): - client_socket, server_socket = self.getConnectedSockets( - test_client, test_server - ) - server_security: grpc_channelz.Security = server_socket.security - client_security: grpc_channelz.Security = client_socket.security - logger.info("Server certs: %s", self.debug_sock_certs(server_security)) - logger.info("Client certs: %s", self.debug_sock_certs(client_security)) - - if mode is self.SecurityMode.MTLS: - self.assertSecurityMtls(client_security, server_security) - elif mode is self.SecurityMode.TLS: - self.assertSecurityTls(client_security, server_security) - elif mode is self.SecurityMode.PLAINTEXT: - self.assertSecurityPlaintext(client_security, server_security) - else: - raise TypeError("Incorrect security mode") - - def assertSecurityMtls( - self, - client_security: grpc_channelz.Security, - server_security: grpc_channelz.Security, - ): - self.assertEqual( - client_security.WhichOneof("model"), - "tls", - msg="(mTLS) Client socket security model must be TLS", - ) - self.assertEqual( - server_security.WhichOneof("model"), - "tls", - msg="(mTLS) Server socket security model must be TLS", - ) - server_tls, client_tls = server_security.tls, client_security.tls - - # Confirm regular TLS: server local cert == client remote cert - self.assertNotEmpty( - client_tls.remote_certificate, - msg="(mTLS) Client remote certificate is missing", - ) - if self.check_local_certs: - self.assertNotEmpty( - server_tls.local_certificate, - msg="(mTLS) Server local certificate is missing", - ) - self.assertEqual( - server_tls.local_certificate, - client_tls.remote_certificate, - msg=( - "(mTLS) Server local certificate must match client's " - "remote certificate" - ), - ) - - # mTLS: server remote cert == client local cert - self.assertNotEmpty( - server_tls.remote_certificate, - msg="(mTLS) Server remote certificate is missing", - ) - if self.check_local_certs: - self.assertNotEmpty( - client_tls.local_certificate, - msg="(mTLS) Client local certificate is missing", - ) - self.assertEqual( - server_tls.remote_certificate, - client_tls.local_certificate, - msg=( - "(mTLS) Server remote certificate must match client's " - "local certificate" - ), - ) - - def assertSecurityTls( - self, - client_security: grpc_channelz.Security, - server_security: grpc_channelz.Security, - ): - self.assertEqual( - client_security.WhichOneof("model"), - "tls", - msg="(TLS) Client socket security model must be TLS", - ) - self.assertEqual( - server_security.WhichOneof("model"), - "tls", - msg="(TLS) Server socket security model must be TLS", - ) - server_tls, client_tls = server_security.tls, client_security.tls - - # Regular TLS: server local cert == client remote cert - self.assertNotEmpty( - client_tls.remote_certificate, - msg="(TLS) Client remote certificate is missing", - ) - if self.check_local_certs: - self.assertNotEmpty( - server_tls.local_certificate, - msg="(TLS) Server local certificate is missing", - ) - self.assertEqual( - server_tls.local_certificate, - client_tls.remote_certificate, - msg=( - "(TLS) Server local certificate must match client " - "remote certificate" - ), - ) - - # mTLS must not be used - self.assertEmpty( - server_tls.remote_certificate, - msg=( - "(TLS) Server remote certificate must be empty in TLS mode. " - "Is server security incorrectly configured for mTLS?" - ), - ) - self.assertEmpty( - client_tls.local_certificate, - msg=( - "(TLS) Client local certificate must be empty in TLS mode. " - "Is client security incorrectly configured for mTLS?" - ), - ) - - def assertSecurityPlaintext(self, client_security, server_security): - server_tls, client_tls = server_security.tls, client_security.tls - # Not TLS - self.assertEmpty( - server_tls.local_certificate, - msg="(Plaintext) Server local certificate must be empty.", - ) - self.assertEmpty( - client_tls.local_certificate, - msg="(Plaintext) Client local certificate must be empty.", - ) - - # Not mTLS - self.assertEmpty( - server_tls.remote_certificate, - msg="(Plaintext) Server remote certificate must be empty.", - ) - self.assertEmpty( - client_tls.local_certificate, - msg="(Plaintext) Client local certificate must be empty.", - ) - - def assertClientCannotReachServerRepeatedly( - self, - test_client: XdsTestClient, - *, - times: Optional[int] = None, - delay: Optional[_timedelta] = None, - ): - """ - Asserts that the client repeatedly cannot reach the server. - - With negative tests we can't be absolutely certain expected failure - state is not caused by something else. - To mitigate for this, we repeat the checks several times, and expect - all of them to succeed. - - This is useful in case the channel eventually stabilizes, and RPCs pass. - - Args: - test_client: An instance of XdsTestClient - times: Optional; A positive number of times to confirm that - the server is unreachable. Defaults to `3` attempts. - delay: Optional; Specifies how long to wait before the next check. - Defaults to `10` seconds. - """ - if times is None or times < 1: - times = 3 - if delay is None: - delay = _timedelta(seconds=10) - - for i in range(1, times + 1): - self.assertClientCannotReachServer(test_client) - if i < times: - logger.info( - "Check %s passed, waiting %s before the next check", - i, - delay, - ) - time.sleep(delay.total_seconds()) - - def assertClientCannotReachServer(self, test_client: XdsTestClient): - self.assertClientChannelFailed(test_client) - self.assertFailedRpcs(test_client) - - def assertClientChannelFailed(self, test_client: XdsTestClient): - channel = test_client.wait_for_server_channel_state( - state=_ChannelState.TRANSIENT_FAILURE - ) - subchannels = list( - test_client.channelz.list_channel_subchannels(channel) - ) - self.assertLen( - subchannels, - 1, - msg=( - "Client channel must have exactly one subchannel " - "in state TRANSIENT_FAILURE." - ), - ) - - @staticmethod - def getConnectedSockets( - test_client: XdsTestClient, test_server: XdsTestServer - ) -> Tuple[grpc_channelz.Socket, grpc_channelz.Socket]: - client_sock = test_client.get_active_server_channel_socket() - server_sock = test_server.get_server_socket_matching_client(client_sock) - return client_sock, server_sock - - @classmethod - def debug_sock_certs(cls, security: grpc_channelz.Security): - if security.WhichOneof("model") == "other": - return f"other: <{security.other.name}={security.other.value}>" - - return ( - f"local: <{cls.debug_cert(security.tls.local_certificate)}>, " - f"remote: <{cls.debug_cert(security.tls.remote_certificate)}>" - ) - - @staticmethod - def debug_cert(cert): - if not cert: - return "missing" - sha1 = hashlib.sha1(cert) - return f"sha1={sha1.hexdigest()}, len={len(cert)}" diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py deleted file mode 100644 index becf41ec3d0..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py +++ /dev/null @@ -1,378 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A test framework built for urlMap related xDS test cases.""" - -import functools -import inspect -from typing import Any, Iterable, Mapping, Tuple - -from absl import flags -from absl import logging - -from framework import xds_flags -from framework import xds_k8s_flags -import framework.helpers.rand -from framework.infrastructure import gcp -from framework.infrastructure import k8s -from framework.infrastructure import traffic_director -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner - -flags.adopt_module_key_flags(xds_flags) -flags.adopt_module_key_flags(xds_k8s_flags) - -STRATEGY = flags.DEFINE_enum( - "strategy", - default="reuse", - enum_values=["create", "keep", "reuse"], - help="Strategy of GCP resources management", -) - -# Type alias -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner -_KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -UrlMapType = Any -HostRule = Any -PathMatcher = Any - - -class _UrlMapChangeAggregator: - """Where all the urlMap change happens.""" - - def __init__(self, url_map_name: str): - self._map = { - "name": url_map_name, - "defaultService": GcpResourceManager().default_backend_service(), - "hostRules": [], - "pathMatchers": [], - } - - def get_map(self) -> UrlMapType: - return self._map - - def apply_change(self, test_case: "XdsUrlMapTestCase") -> None: - logging.info( - "Apply urlMap change for test case: %s.%s", - test_case.short_module_name, - test_case.__name__, - ) - url_map_parts = test_case.url_map_change( - *self._get_test_case_url_map(test_case) - ) - self._set_test_case_url_map(*url_map_parts) - - @staticmethod - def _get_test_case_url_map( - test_case: "XdsUrlMapTestCase", - ) -> Tuple[HostRule, PathMatcher]: - host_rule = { - "hosts": [test_case.hostname()], - "pathMatcher": test_case.path_matcher_name(), - } - path_matcher = { - "name": test_case.path_matcher_name(), - "defaultService": GcpResourceManager().default_backend_service(), - } - return host_rule, path_matcher - - def _set_test_case_url_map( - self, host_rule: HostRule, path_matcher: PathMatcher - ) -> None: - self._map["hostRules"].append(host_rule) - self._map["pathMatchers"].append(path_matcher) - - -def _package_flags() -> Mapping[str, Any]: - """Automatically parse Abseil flags into a dictionary. - - Abseil flag is only available after the Abseil app initialization. If we use - __new__ in our metaclass, the flag value parse will happen during the - initialization of modules, hence will fail. That's why we are using __call__ - to inject metaclass magics, and the flag parsing will be delayed until the - class is about to be instantiated. - """ - res = {} - for flag_module in [xds_flags, xds_k8s_flags]: - for key, value in inspect.getmembers(flag_module): - if isinstance(value, flags.FlagHolder): - res[key.lower()] = value.value - res["strategy"] = STRATEGY.value - return res - - -class _MetaSingletonAndAbslFlags(type): - """Ensures singleton and injects flag values.""" - - # Allow different subclasses to create different singletons. - _instances = {} - # But we only parse Abseil flags once. - _flags = None - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - if cls._flags is None: - cls._flags = _package_flags() - obj = super().__call__(cls._flags, *args, **kwargs) - cls._instances[cls] = obj - return obj - return cls._instances[cls] - - -class GcpResourceManager(metaclass=_MetaSingletonAndAbslFlags): - """Manages the lifecycle of GCP resources. - - The GCP resources including: - - 3 K8s deployment (client, default backends, alternative backends) - - Full set of the Traffic Director stuff - - Merged gigantic urlMap from all imported test cases - - All resources are intended to be used across test cases and multiple runs - (except the client K8s deployment). - """ - - # This class dynamically set, so disable "no-member" check. - # pylint: disable=no-member - - def __init__(self, absl_flags: Mapping[str, Any] = None): - if absl_flags is not None: - for key in absl_flags: - setattr(self, key, absl_flags[key]) - # Pick a client_namespace_suffix if not set - if getattr(self, "resource_suffix", None) is None: - self.resource_suffix = "" - else: - raise NotImplementedError( - "Predefined resource_suffix is not supported for UrlMap tests" - ) - logging.info( - "GcpResourceManager: resource prefix=%s, suffix=%s", - self.resource_prefix, - self.resource_suffix, - ) - - # Must be called before KubernetesApiManager or GcpApiManager init. - xds_flags.set_socket_default_timeout_from_flag() - - # API managers - self.k8s_api_manager = k8s.KubernetesApiManager(self.kube_context) - self.gcp_api_manager = gcp.api.GcpApiManager() - self.td = traffic_director.TrafficDirectorManager( - self.gcp_api_manager, - self.project, - resource_prefix=self.resource_prefix, - resource_suffix=(self.resource_suffix or ""), - network=self.network, - compute_api_version=self.compute_api_version, - ) - # Kubernetes namespace - self.k8s_namespace = k8s.KubernetesNamespace( - self.k8s_api_manager, self.resource_prefix - ) - # Kubernetes Test Servers - self.test_server_runner = _KubernetesServerRunner( - self.k8s_namespace, - deployment_name=self.server_name, - image_name=self.server_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - enable_workload_identity=self.enable_workload_identity, - ) - self.test_server_alternative_runner = _KubernetesServerRunner( - self.k8s_namespace, - deployment_name=self.server_name + "-alternative", - image_name=self.server_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - enable_workload_identity=self.enable_workload_identity, - reuse_namespace=True, - ) - self.test_server_affinity_runner = _KubernetesServerRunner( - self.k8s_namespace, - deployment_name=self.server_name + "-affinity", - image_name=self.server_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - enable_workload_identity=self.enable_workload_identity, - reuse_namespace=True, - ) - logging.info("Strategy of GCP resources management: %s", self.strategy) - - def create_test_client_runner(self): - if self.resource_suffix: - client_namespace_suffix = self.resource_suffix - else: - client_namespace_suffix = ( - framework.helpers.rand.random_resource_suffix() - ) - logging.info( - "GcpResourceManager: client_namespace_suffix=%s", - client_namespace_suffix, - ) - # Kubernetes Test Client - namespace_name = _KubernetesClientRunner.make_namespace_name( - self.resource_prefix, client_namespace_suffix - ) - return _KubernetesClientRunner( - k8s.KubernetesNamespace(self.k8s_api_manager, namespace_name), - deployment_name=self.client_name, - image_name=self.client_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - enable_workload_identity=self.enable_workload_identity, - stats_port=self.client_port, - ) - - def _pre_cleanup(self): - # Cleanup existing debris - logging.info("GcpResourceManager: pre clean-up") - self.td.cleanup(force=True) - self.test_server_runner.delete_namespace() - - def setup(self, test_case_classes: Iterable["XdsUrlMapTestCase"]) -> None: - if self.strategy not in ["create", "keep"]: - logging.info( - "GcpResourceManager: skipping setup for strategy [%s]", - self.strategy, - ) - return - # Clean up debris from previous runs - self._pre_cleanup() - # Start creating GCP resources - logging.info("GcpResourceManager: start setup") - # Firewall - if self.ensure_firewall: - self.td.create_firewall_rule( - allowed_ports=self.firewall_allowed_ports - ) - # Health Checks - self.td.create_health_check() - # Backend Services - self.td.create_backend_service() - self.td.create_alternative_backend_service() - self.td.create_affinity_backend_service() - # Construct UrlMap from test classes - aggregator = _UrlMapChangeAggregator( - url_map_name=self.td.make_resource_name(self.td.URL_MAP_NAME) - ) - for test_case_class in test_case_classes: - aggregator.apply_change(test_case_class) - final_url_map = aggregator.get_map() - # UrlMap - self.td.create_url_map_with_content(final_url_map) - # Target Proxy - self.td.create_target_proxy() - # Forwarding Rule - self.td.create_forwarding_rule(self.server_xds_port) - # Kubernetes Test Server - self.test_server_runner.run( - test_port=self.server_port, - maintenance_port=self.server_maintenance_port, - ) - # Kubernetes Test Server Alternative - self.test_server_alternative_runner.run( - test_port=self.server_port, - maintenance_port=self.server_maintenance_port, - ) - # Kubernetes Test Server Affinity. 3 endpoints to test that only the - # picked sub-channel is connected. - self.test_server_affinity_runner.run( - test_port=self.server_port, - maintenance_port=self.server_maintenance_port, - replica_count=3, - ) - # Add backend to default backend service - neg_name, neg_zones = self.k8s_namespace.parse_service_neg_status( - self.test_server_runner.service_name, self.server_port - ) - self.td.backend_service_add_neg_backends(neg_name, neg_zones) - # Add backend to alternative backend service - ( - neg_name_alt, - neg_zones_alt, - ) = self.k8s_namespace.parse_service_neg_status( - self.test_server_alternative_runner.service_name, self.server_port - ) - self.td.alternative_backend_service_add_neg_backends( - neg_name_alt, neg_zones_alt - ) - # Add backend to affinity backend service - ( - neg_name_affinity, - neg_zones_affinity, - ) = self.k8s_namespace.parse_service_neg_status( - self.test_server_affinity_runner.service_name, self.server_port - ) - self.td.affinity_backend_service_add_neg_backends( - neg_name_affinity, neg_zones_affinity - ) - # Wait for healthy backends - self.td.wait_for_backends_healthy_status() - self.td.wait_for_alternative_backends_healthy_status() - self.td.wait_for_affinity_backends_healthy_status() - - def cleanup(self) -> None: - if self.strategy not in ["create"]: - logging.info( - "GcpResourceManager: skipping tear down for strategy [%s]", - self.strategy, - ) - return - logging.info("GcpResourceManager: start tear down") - if hasattr(self, "td"): - self.td.cleanup(force=True) - if hasattr(self, "test_server_runner"): - self.test_server_runner.cleanup(force=True) - if hasattr(self, "test_server_alternative_runner"): - self.test_server_alternative_runner.cleanup( - force=True, force_namespace=True - ) - if hasattr(self, "test_server_affinity_runner"): - self.test_server_affinity_runner.cleanup( - force=True, force_namespace=True - ) - - @functools.lru_cache(None) - def default_backend_service(self) -> str: - """Returns default backend service URL.""" - self.td.load_backend_service() - return self.td.backend_service.url - - @functools.lru_cache(None) - def alternative_backend_service(self) -> str: - """Returns alternative backend service URL.""" - self.td.load_alternative_backend_service() - return self.td.alternative_backend_service.url - - @functools.lru_cache(None) - def affinity_backend_service(self) -> str: - """Returns affinity backend service URL.""" - self.td.load_affinity_backend_service() - return self.td.affinity_backend_service.url diff --git a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py b/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py deleted file mode 100644 index 25ef9ee9a6b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_testcase.py +++ /dev/null @@ -1,607 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A test framework built for urlMap related xDS test cases.""" - -import abc -from dataclasses import dataclass -import datetime -import json -import os -import re -import sys -import time -from typing import Any, Iterable, Mapping, Optional, Tuple -import unittest - -from absl import flags -from absl import logging -from google.protobuf import json_format -import grpc - -from framework import xds_k8s_testcase -from framework import xds_url_map_test_resources -from framework.helpers import grpc as helpers_grpc -from framework.helpers import retryers -from framework.helpers import skips -from framework.infrastructure import k8s -from framework.test_app import client_app -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_cases import base_testcase - -# Load existing flags -flags.adopt_module_key_flags(xds_k8s_testcase) -flags.adopt_module_key_flags(xds_url_map_test_resources) - -# Define urlMap specific flags -QPS = flags.DEFINE_integer("qps", default=25, help="The QPS client is sending") - -# Test configs -_URL_MAP_PROPAGATE_TIMEOUT_SEC = 600 -# With the per-run IAM change, the first xDS response has a several minutes -# delay. We want to increase the interval, reduce the log spam. -_URL_MAP_PROPAGATE_CHECK_INTERVAL_SEC = 15 -URL_MAP_TESTCASE_FILE_SUFFIX = "_test.py" -_CLIENT_CONFIGURE_WAIT_SEC = 2 - -# Type aliases -XdsTestClient = client_app.XdsTestClient -GcpResourceManager = xds_url_map_test_resources.GcpResourceManager -HostRule = xds_url_map_test_resources.HostRule -PathMatcher = xds_url_map_test_resources.PathMatcher -_KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -JsonType = Any -_timedelta = datetime.timedelta - -# ProtoBuf translatable RpcType enums -RpcTypeUnaryCall = "UNARY_CALL" -RpcTypeEmptyCall = "EMPTY_CALL" - - -def _split_camel(s: str, delimiter: str = "-") -> str: - """Turn camel case name to snake-case-like name.""" - return "".join( - delimiter + c.lower() if c.isupper() else c for c in s - ).lstrip(delimiter) - - -class DumpedXdsConfig(dict): - """A convenience class to check xDS config. - - Feel free to add more pre-compute fields. - """ - - def __init__(self, xds_json: JsonType): # pylint: disable=too-many-branches - super().__init__(xds_json) - self.json_config = xds_json - self.lds = None - self.rds = None - self.rds_version = None - self.cds = [] - self.eds = [] - self.endpoints = [] - for xds_config in self.get("xdsConfig", []): - try: - if "listenerConfig" in xds_config: - self.lds = xds_config["listenerConfig"]["dynamicListeners"][ - 0 - ]["activeState"]["listener"] - elif "routeConfig" in xds_config: - self.rds = xds_config["routeConfig"]["dynamicRouteConfigs"][ - 0 - ]["routeConfig"] - self.rds_version = xds_config["routeConfig"][ - "dynamicRouteConfigs" - ][0]["versionInfo"] - elif "clusterConfig" in xds_config: - for cluster in xds_config["clusterConfig"][ - "dynamicActiveClusters" - ]: - self.cds.append(cluster["cluster"]) - elif "endpointConfig" in xds_config: - for endpoint in xds_config["endpointConfig"][ - "dynamicEndpointConfigs" - ]: - self.eds.append(endpoint["endpointConfig"]) - # TODO(lidiz) reduce the catch to LookupError - except Exception as e: # pylint: disable=broad-except - logging.debug( - "Parsing dumped xDS config failed with %s: %s", type(e), e - ) - for generic_xds_config in self.get("genericXdsConfigs", []): - try: - if re.search(r"\.Listener$", generic_xds_config["typeUrl"]): - self.lds = generic_xds_config["xdsConfig"] - elif re.search( - r"\.RouteConfiguration$", generic_xds_config["typeUrl"] - ): - self.rds = generic_xds_config["xdsConfig"] - self.rds_version = generic_xds_config["versionInfo"] - elif re.search(r"\.Cluster$", generic_xds_config["typeUrl"]): - self.cds.append(generic_xds_config["xdsConfig"]) - elif re.search( - r"\.ClusterLoadAssignment$", generic_xds_config["typeUrl"] - ): - self.eds.append(generic_xds_config["xdsConfig"]) - # TODO(lidiz) reduce the catch to LookupError - except Exception as e: # pylint: disable=broad-except - logging.debug( - "Parsing dumped xDS config failed with %s: %s", type(e), e - ) - for endpoint_config in self.eds: - for endpoint in endpoint_config.get("endpoints", {}): - for lb_endpoint in endpoint.get("lbEndpoints", {}): - try: - if lb_endpoint["healthStatus"] == "HEALTHY": - self.endpoints.append( - "%s:%s" - % ( - lb_endpoint["endpoint"]["address"][ - "socketAddress" - ]["address"], - lb_endpoint["endpoint"]["address"][ - "socketAddress" - ]["portValue"], - ) - ) - # TODO(lidiz) reduce the catch to LookupError - except Exception as e: # pylint: disable=broad-except - logging.debug( - "Parse endpoint failed with %s: %s", type(e), e - ) - - def __str__(self) -> str: - return json.dumps(self, indent=2) - - -class RpcDistributionStats: - """A convenience class to check RPC distribution. - - Feel free to add more pre-compute fields. - """ - - num_failures: int - num_oks: int - default_service_rpc_count: int - alternative_service_rpc_count: int - unary_call_default_service_rpc_count: int - empty_call_default_service_rpc_count: int - unary_call_alternative_service_rpc_count: int - empty_call_alternative_service_rpc_count: int - - def __init__(self, json_lb_stats: JsonType): - self.num_failures = json_lb_stats.get("numFailures", 0) - - self.num_peers = 0 - self.num_oks = 0 - self.default_service_rpc_count = 0 - self.alternative_service_rpc_count = 0 - self.unary_call_default_service_rpc_count = 0 - self.empty_call_default_service_rpc_count = 0 - self.unary_call_alternative_service_rpc_count = 0 - self.empty_call_alternative_service_rpc_count = 0 - self.raw = json_lb_stats - - if "rpcsByPeer" in json_lb_stats: - self.num_peers = len(json_lb_stats["rpcsByPeer"]) - if "rpcsByMethod" in json_lb_stats: - for rpc_type in json_lb_stats["rpcsByMethod"]: - for peer in json_lb_stats["rpcsByMethod"][rpc_type][ - "rpcsByPeer" - ]: - count = json_lb_stats["rpcsByMethod"][rpc_type][ - "rpcsByPeer" - ][peer] - self.num_oks += count - if rpc_type == "UnaryCall": - if "alternative" in peer: - self.unary_call_alternative_service_rpc_count = ( - count - ) - self.alternative_service_rpc_count += count - else: - self.unary_call_default_service_rpc_count = count - self.default_service_rpc_count += count - else: - if "alternative" in peer: - self.empty_call_alternative_service_rpc_count = ( - count - ) - self.alternative_service_rpc_count += count - else: - self.empty_call_default_service_rpc_count = count - self.default_service_rpc_count += count - - -@dataclass -class ExpectedResult: - """Describes the expected result of assertRpcStatusCode method below.""" - - rpc_type: str = RpcTypeUnaryCall - status_code: grpc.StatusCode = grpc.StatusCode.OK - ratio: float = 1 - - -class _MetaXdsUrlMapTestCase(type): - """Tracking test case subclasses.""" - - # Automatic discover of all subclasses - _test_case_classes = [] - _test_case_names = set() - # Keep track of started and finished test cases, so we know when to setup - # and tear down GCP resources. - _started_test_cases = set() - _finished_test_cases = set() - - def __new__( - cls, name: str, bases: Iterable[Any], attrs: Mapping[str, Any] - ) -> Any: - # Hand over the tracking objects - attrs["test_case_classes"] = cls._test_case_classes - attrs["test_case_names"] = cls._test_case_names - attrs["started_test_cases"] = cls._started_test_cases - attrs["finished_test_cases"] = cls._finished_test_cases - # Handle the test name reflection - module_name = os.path.split(sys.modules[attrs["__module__"]].__file__)[ - -1 - ] - if module_name.endswith(URL_MAP_TESTCASE_FILE_SUFFIX): - module_name = module_name.replace(URL_MAP_TESTCASE_FILE_SUFFIX, "") - attrs["short_module_name"] = module_name.replace("_", "-") - # Create the class and track - new_class = type.__new__(cls, name, bases, attrs) - if name.startswith("Test"): - cls._test_case_names.add(name) - cls._test_case_classes.append(new_class) - else: - logging.debug("Skipping test case class: %s", name) - return new_class - - -class XdsUrlMapTestCase( - base_testcase.BaseTestCase, metaclass=_MetaXdsUrlMapTestCase -): - """XdsUrlMapTestCase is the base class for urlMap related tests. - - The subclass is expected to implement 3 methods: - - - url_map_change: Updates the urlMap components for this test case - - xds_config_validate: Validates if the client received legit xDS configs - - rpc_distribution_validate: Validates if the routing behavior is correct - """ - - test_client_runner: Optional[_KubernetesClientRunner] = None - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - """Allow the test case to decide whether it supports the given config. - - Returns: - A bool indicates if the given config is supported. - """ - del config - return True - - @staticmethod - def client_init_config(rpc: str, metadata: str) -> Tuple[str, str]: - """Updates the initial RPC configs for this test case. - - Each test case will start a test client. The client takes RPC configs - and starts to send RPCs immediately. The config returned by this - function will be used to replace the default configs. - - The default configs are passed in as arguments, so this method can - modify part of them. - - Args: - rpc: The default rpc config, specifying RPCs to send, format - 'UnaryCall,EmptyCall' - metadata: The metadata config, specifying metadata to send with each - RPC, format 'EmptyCall:key1:value1,UnaryCall:key2:value2'. - - Returns: - A tuple contains the updated rpc and metadata config. - """ - return rpc, metadata - - @staticmethod - @abc.abstractmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - """Updates the dedicated urlMap components for this test case. - - Each test case will have a dedicated HostRule, where the hostname is - generated from the test case name. The HostRule will be linked to a - PathMatcher, where stores the routing logic. - - Args: - host_rule: A HostRule GCP resource as a JSON dict. - path_matcher: A PathMatcher GCP resource as a JSON dict. - - Returns: - A tuple contains the updated version of given HostRule and - PathMatcher. - """ - - @abc.abstractmethod - def xds_config_validate(self, xds_config: DumpedXdsConfig) -> None: - """Validates received xDS config, if anything is wrong, raise. - - This stage only ends when the control plane failed to send a valid - config within a given time range, like 600s. - - Args: - xds_config: A DumpedXdsConfig instance can be used as a JSON dict, - but also provides helper fields for commonly checked xDS config. - """ - - @abc.abstractmethod - def rpc_distribution_validate(self, test_client: XdsTestClient) -> None: - """Validates the routing behavior, if any is wrong, raise. - - Args: - test_client: A XdsTestClient instance for all sorts of end2end testing. - """ - - @classmethod - def hostname(cls): - return "%s.%s:%s" % ( - cls.short_module_name, - _split_camel(cls.__name__), - GcpResourceManager().server_xds_port, - ) - - @classmethod - def path_matcher_name(cls): - # Path matcher name must match r'(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)' - return "%s-%s-pm" % (cls.short_module_name, _split_camel(cls.__name__)) - - @classmethod - def setUpClass(cls): - logging.info("----- Testing %s -----", cls.__name__) - logging.info("Logs timezone: %s", time.localtime().tm_zone) - - # Raises unittest.SkipTest if given client/server/version does not - # support current test case. - xds_k8s_testcase.evaluate_test_config(cls.is_supported) - - # Configure cleanup to run after all tests regardless of - # whether setUpClass failed. - cls.addClassCleanup(cls.cleanupAfterTests) - - if not cls.started_test_cases: - # Create the GCP resource once before the first test start - GcpResourceManager().setup(cls.test_case_classes) - cls.started_test_cases.add(cls.__name__) - - # Create the test case's own client runner with it's own namespace, - # enables concurrent running with other test cases. - cls.test_client_runner = ( - GcpResourceManager().create_test_client_runner() - ) - # Start the client, and allow the test to override the initial RPC config. - rpc, metadata = cls.client_init_config( - rpc="UnaryCall,EmptyCall", metadata="" - ) - cls.test_client = cls.test_client_runner.run( - server_target=f"xds:///{cls.hostname()}", - rpc=rpc, - metadata=metadata, - qps=QPS.value, - print_response=True, - ) - - @classmethod - def cleanupAfterTests(cls): - logging.info("----- TestCase %s teardown -----", cls.__name__) - client_restarts: int = 0 - if cls.test_client_runner: - try: - logging.debug("Getting pods restart times") - client_restarts = cls.test_client_runner.get_pod_restarts( - cls.test_client_runner.deployment - ) - except (retryers.RetryError, k8s.NotFound) as e: - logging.exception(e) - - cls.finished_test_cases.add(cls.__name__) - # Whether to clean up shared pre-provisioned infrastructure too. - # We only do it after all tests are finished. - cleanup_all = cls.finished_test_cases == cls.test_case_names - - # Graceful cleanup: try three times, and don't fail the test on - # a cleanup failure. - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=10), - attempts=3, - log_level=logging.INFO, - ) - try: - retryer(cls._cleanup, cleanup_all) - except retryers.RetryError: - logging.exception("Got error during teardown") - finally: - if hasattr(cls, "test_client_runner") and cls.test_client_runner: - logging.info("----- Test client logs -----") - cls.test_client_runner.logs_explorer_run_history_links() - - # Fail if any of the pods restarted. - error_msg = ( - "Client container unexpectedly restarted" - f" {client_restarts} times during test." - " In most cases, this is caused by the test client app crash." - ) - assert client_restarts == 0, error_msg - - @classmethod - def _cleanup(cls, cleanup_all: bool = False): - if cls.test_client_runner: - cls.test_client_runner.cleanup(force=True, force_namespace=True) - if cleanup_all: - GcpResourceManager().cleanup() - - def _fetch_and_check_xds_config(self): - # TODO(lidiz) find another way to store last seen xDS config - # Cleanup state for this attempt - # pylint: disable=attribute-defined-outside-init - self._xds_json_config = None - # Fetch client config - config = self.test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - self.assertIsNotNone(config) - # Found client config, test it. - self._xds_json_config = json_format.MessageToDict(config) - # pylint: enable=attribute-defined-outside-init - # Execute the child class provided validation logic - self.xds_config_validate(DumpedXdsConfig(self._xds_json_config)) - - def run(self, result: unittest.TestResult = None) -> None: - """Abort this test case if CSDS check is failed. - - This prevents the test runner to waste time on RPC distribution test, - and yields clearer signal. - """ - if result.failures or result.errors: - logging.info("Aborting %s", self.__class__.__name__) - else: - super().run(result) - - def test_client_config(self): - self.test_client.wait_for_active_xds_channel( - xds_server_uri=GcpResourceManager().xds_server_uri, - ) - retryer = retryers.constant_retryer( - wait_fixed=datetime.timedelta( - seconds=_URL_MAP_PROPAGATE_CHECK_INTERVAL_SEC - ), - timeout=datetime.timedelta(seconds=_URL_MAP_PROPAGATE_TIMEOUT_SEC), - logger=logging, - log_level=logging.INFO, - ) - try: - retryer(self._fetch_and_check_xds_config) - finally: - logging.info( - "latest xDS config:\n%s", - GcpResourceManager().td.compute.resource_pretty_format( - self._xds_json_config - ), - ) - - def test_rpc_distribution(self): - self.rpc_distribution_validate(self.test_client) - - @classmethod - def configure_and_send( - cls, - test_client: XdsTestClient, - *, - rpc_types: Iterable[str], - metadata: Optional[Iterable[Tuple[str, str, str]]] = None, - app_timeout: Optional[int] = None, - num_rpcs: int, - ) -> RpcDistributionStats: - test_client.update_config.configure( - rpc_types=rpc_types, metadata=metadata, app_timeout=app_timeout - ) - # Configure RPC might race with get stats RPC on slower machines. - time.sleep(_CLIENT_CONFIGURE_WAIT_SEC) - lb_stats = test_client.get_load_balancer_stats(num_rpcs=num_rpcs) - logging.info( - "[%s] << Received LoadBalancerStatsResponse:\n%s", - test_client.hostname, - helpers_grpc.lb_stats_pretty(lb_stats), - ) - return RpcDistributionStats(json_format.MessageToDict(lb_stats)) - - def assertNumEndpoints(self, xds_config: DumpedXdsConfig, k: int) -> None: - self.assertLen( - xds_config.endpoints, - k, - ( - "insufficient endpoints in EDS:" - f" want={k} seen={xds_config.endpoints}" - ), - ) - - def assertRpcStatusCode( # pylint: disable=too-many-locals - self, - test_client: XdsTestClient, - *, - expected: Iterable[ExpectedResult], - length: int, - tolerance: float, - ) -> None: - """Assert the distribution of RPC statuses over a period of time.""" - # Sending with pre-set QPS for a period of time - before_stats = test_client.get_load_balancer_accumulated_stats() - logging.info( - ( - "Received LoadBalancerAccumulatedStatsResponse from test client" - " %s: before:\n%s" - ), - test_client.hostname, - helpers_grpc.accumulated_stats_pretty(before_stats), - ) - time.sleep(length) - after_stats = test_client.get_load_balancer_accumulated_stats() - logging.info( - ( - "Received LoadBalancerAccumulatedStatsResponse from test client" - " %s: after: \n%s" - ), - test_client.hostname, - helpers_grpc.accumulated_stats_pretty(after_stats), - ) - - # Validate the diff - for expected_result in expected: - rpc = expected_result.rpc_type - status = expected_result.status_code.value[0] - # Compute observation - # ProtoBuf messages has special magic dictionary that we don't need - # to catch exceptions: - # https://developers.google.com/protocol-buffers/docs/reference/python-generated#undefined - seen_after = after_stats.stats_per_method[rpc].result[status] - seen_before = before_stats.stats_per_method[rpc].result[status] - seen = seen_after - seen_before - # Compute total number of RPC started - stats_per_method_after = after_stats.stats_per_method.get( - rpc, {} - ).result.items() - total_after = sum( - x[1] for x in stats_per_method_after - ) # (status_code, count) - stats_per_method_before = before_stats.stats_per_method.get( - rpc, {} - ).result.items() - total_before = sum( - x[1] for x in stats_per_method_before - ) # (status_code, count) - total = total_after - total_before - # Compute and validate the number - want = total * expected_result.ratio - diff_ratio = abs(seen - want) / total - self.assertLessEqual( - diff_ratio, - tolerance, - ( - f"Expect rpc [{rpc}] to return " - f"[{expected_result.status_code}] at " - f"{expected_result.ratio:.2f} ratio: " - f"seen={seen} want={want} total={total} " - f"diff_ratio={diff_ratio:.4f} > {tolerance:.2f}" - ), - ) diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml deleted file mode 100644 index 4803ecfa709..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client-secure.deployment.yaml +++ /dev/null @@ -1,94 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deployment_name} - namespace: ${namespace_name} - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test -spec: - replicas: 1 - selector: - matchLabels: - app: ${deployment_name} - deployment_id: ${deployment_id} - template: - metadata: - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test - annotations: - security.cloud.google.com/use-workload-certificates: "" - spec: - serviceAccountName: ${service_account_name} - containers: - - name: ${deployment_name} - image: ${image_name} - imagePullPolicy: Always - startupProbe: - tcpSocket: - port: ${stats_port} - periodSeconds: 3 - ## Extend the number of probes well beyond the duration of the test - ## driver waiting for the container to start. - failureThreshold: 1000 - args: - - "--server=${server_target}" - - "--stats_port=${stats_port}" - - "--secure_mode=${secure_mode}" - - "--qps=${qps}" - - "--rpc=${rpc}" - - "--print_response=${print_response}" - ports: - - containerPort: ${stats_port} - env: - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_V3_SUPPORT - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - resources: - limits: - cpu: 800m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi - initContainers: - - name: grpc-td-init - image: ${td_bootstrap_image} - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - % if config_mesh: - - "--config-mesh-experimental=${config_mesh}" - % endif - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml deleted file mode 100644 index 13fad0f8cd3..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/client.deployment.yaml +++ /dev/null @@ -1,104 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deployment_name} - namespace: ${namespace_name} - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test -spec: - replicas: 1 - selector: - matchLabels: - app: ${deployment_name} - deployment_id: ${deployment_id} - template: - metadata: - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test - spec: - % if service_account_name: - serviceAccountName: ${service_account_name} - % endif - containers: - - name: ${deployment_name} - image: ${image_name} - imagePullPolicy: Always - startupProbe: - tcpSocket: - port: ${stats_port} - periodSeconds: 3 - ## Extend the number of probes well beyond the duration of the test - ## driver waiting for the container to start. - failureThreshold: 1000 - args: - - "--server=${server_target}" - - "--stats_port=${stats_port}" - - "--qps=${qps}" - - "--rpc=${rpc}" - - "--metadata=${metadata}" - - "--print_response=${print_response}" - % if enable_csm_observability: - - "--enable_csm_observability" - % endif - ports: - - containerPort: ${stats_port} - env: - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY - value: "true" - - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - resources: - limits: - cpu: 800m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi - initContainers: - - name: grpc-td-init - image: ${td_bootstrap_image} - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - % if config_mesh: - - "--config-mesh-experimental=${config_mesh}" - % endif - % if generate_mesh_id: - - "--generate-mesh-id-experimental" - % endif - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/csm/pod-monitoring.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/csm/pod-monitoring.yaml deleted file mode 100644 index 3204f0f9a34..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/csm/pod-monitoring.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: monitoring.googleapis.com/v1 -kind: PodMonitoring -metadata: - namespace: ${namespace_name} - name: ${pod_monitoring_name} - labels: - owner: xds-k8s-interop-test -spec: - selector: - matchLabels: - deployment_id: ${deployment_id} - endpoints: - - port: 9464 - interval: 10s diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/backend_policy.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/backend_policy.yaml deleted file mode 100644 index 872b4ef7689..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/backend_policy.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -kind: GCPBackendPolicy -apiVersion: networking.gke.io/v1 -metadata: - name: ${be_policy_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test -spec: - targetRef: - group: "" - kind: Service - name: ${service_name} - default: - connectionDraining: - drainingTimeoutSec: 600 -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/frontend_service.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/frontend_service.yaml deleted file mode 100644 index 9948ec12063..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/frontend_service.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: ${service_name} - namespace: ${namespace_name} -spec: - ports: - - port: 8080 - targetPort: 8080 diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_grpc.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_grpc.yaml deleted file mode 100644 index a023f290bf5..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_grpc.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -kind: GRPCRoute -apiVersion: gateway.networking.k8s.io/v1alpha2 -metadata: - name: ${route_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test -spec: - parentRefs: - - name: ${mesh_name} - namespace: ${namespace_name} - group: net.gke.io - kind: TDMesh - hostnames: - - ${xds_server_uri} - rules: - - backendRefs: - - name: ${service_name} - port: ${test_port} - namespace: ${namespace_name} -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http.yaml deleted file mode 100644 index 0890a99b4d4..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -kind: HTTPRoute -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: ${route_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test -spec: - parentRefs: - - name: ${frontend_service_name} - namespace: ${namespace_name} - group: "" - kind: Service - rules: - - matches: - - path: - type: Exact - value: /grpc.testing.TestService/UnaryCall - backendRefs: - - name: ${service_name} - port: 8080 -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http_ssafilter.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http_ssafilter.yaml deleted file mode 100644 index 5460ad65090..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/route_http_ssafilter.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -kind: HTTPRoute -apiVersion: gateway.networking.k8s.io/v1beta1 -metadata: - name: ${route_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test -spec: - parentRefs: - - name: ${frontend_service_name} - namespace: ${namespace_name} - group: "" - kind: Service - rules: - - matches: - - path: - type: Exact - value: /grpc.testing.TestService/UnaryCall - filters: - - type: ExtensionRef - extensionRef: - group: networking.gke.io - kind: GCPSessionAffinityFilter - name: ssa-filter - backendRefs: - - name: ${service_name} - port: 8080 -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/service.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/service.yaml deleted file mode 100644 index 7a4164ce8bb..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: ${service_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test -spec: - type: ClusterIP - selector: - app: ${deployment_name} - ports: - - port: ${test_port} - protocol: TCP - targetPort: ${test_port} -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_filter.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_filter.yaml deleted file mode 100644 index e0370561dd4..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_filter.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GCPSessionAffinityFilter -metadata: - name: ${session_affinity_filter_name} - namespace: ${namespace_name} -spec: - statefulGeneratedCookie: - cookieTtlSeconds: 50 -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_route.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_route.yaml deleted file mode 100644 index 7042b54bd51..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_route.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GCPSessionAffinityPolicy -metadata: - name: ${session_affinity_policy_name} - namespace: ${namespace_name} -spec: - statefulGeneratedCookie: - cookieTtlSeconds: 50 - targetRef: - name: ${route_name} - group: gateway.networking.k8s.io - kind: HTTPRoute - namespace: ${namespace_name} -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_service.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_service.yaml deleted file mode 100644 index ccc45790bfb..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/gamma/session_affinity_policy_service.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: networking.gke.io/v1 -kind: GCPSessionAffinityPolicy -metadata: - name: ${session_affinity_policy_name} - namespace: ${namespace_name} -spec: - statefulGeneratedCookie: - cookieTtlSeconds: 50 - targetRef: - name: ${service_name} - kind: Service - namespace: ${namespace_name} - group: "" -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/namespace.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/namespace.yaml deleted file mode 100644 index 8b8153a2042..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/namespace.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ${namespace_name} - labels: - name: ${namespace_name} - owner: xds-k8s-interop-test -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml deleted file mode 100644 index ab3b1599e8f..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server-secure.deployment.yaml +++ /dev/null @@ -1,95 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deployment_name} - namespace: ${namespace_name} - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test -spec: - replicas: ${replica_count} - selector: - matchLabels: - app: ${deployment_name} - deployment_id: ${deployment_id} - template: - metadata: - annotations: - security.cloud.google.com/use-workload-certificates: "" - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test - spec: - serviceAccountName: ${service_account_name} - containers: - - name: ${deployment_name} - image: ${image_name} - imagePullPolicy: Always - startupProbe: - tcpSocket: - port: ${maintenance_port} - periodSeconds: 3 - ## Extend the number of probes well beyond the duration of the test - ## driver waiting for the container to start. - failureThreshold: 1000 - args: - - "--port=${test_port}" - - "--maintenance_port=${maintenance_port}" - - "--secure_mode=${secure_mode}" - ports: - - containerPort: ${test_port} - - containerPort: ${maintenance_port} - env: - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_V3_SUPPORT - value: "true" - ## TODO(sergiitk): this should be conditional for if version < v1.37.x - - name: GRPC_XDS_EXPERIMENTAL_NEW_SERVER_API - value: "true" - - name: GRPC_XDS_EXPERIMENTAL_RBAC - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - resources: - limits: - cpu: 800m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi - initContainers: - - name: grpc-td-init - image: ${td_bootstrap_image} - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - - "--node-metadata=app=${namespace_name}-${deployment_name}" - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml deleted file mode 100644 index de823cc6f53..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.deployment.yaml +++ /dev/null @@ -1,101 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${deployment_name} - namespace: ${namespace_name} - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test -spec: - replicas: ${replica_count} - selector: - matchLabels: - app: ${deployment_name} - deployment_id: ${deployment_id} - template: - metadata: - labels: - app: ${deployment_name} - deployment_id: ${deployment_id} - owner: xds-k8s-interop-test - spec: - % if service_account_name: - serviceAccountName: ${service_account_name} - % endif - % if termination_grace_period_seconds: - terminationGracePeriodSeconds: ${termination_grace_period_seconds} - % endif - containers: - - name: ${deployment_name} - image: ${image_name} - imagePullPolicy: Always - startupProbe: - tcpSocket: - port: ${test_port} - periodSeconds: 3 - ## Extend the number of probes well beyond the duration of the test - ## driver waiting for the container to start. - failureThreshold: 1000 - args: - - "--port=${test_port}" - % if enable_csm_observability: - - "--enable_csm_observability" - % endif - ports: - - containerPort: ${test_port} - env: - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_XDS_EXPERIMENTAL_V3_SUPPORT - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST - value: "true" - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - resources: - limits: - cpu: 800m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi - % if pre_stop_hook: - lifecycle: - preStop: - exec: - command: ["tail", "-f", "/dev/null"] - % endif - initContainers: - - name: grpc-td-init - image: ${td_bootstrap_image} - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=${network}" - % if xds_server_uri: - - "--xds-server-uri=${xds_server_uri}" - % endif - % if bootstrap_version=="v0.11.0" or bootstrap_version=="v0.12.0": - - "--node-metadata-experimental=app=${namespace_name}-${deployment_name}" - % else: - - "--node-metadata=app=${namespace_name}-${deployment_name}" - % endif - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.service.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.service.yaml deleted file mode 100644 index 376de175015..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/server.service.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: ${service_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test - annotations: - cloud.google.com/neg: '{"exposed_ports": {"${test_port}":{"name":"${neg_name}"}}}' -spec: - type: ClusterIP - selector: - app: ${deployment_name} - ports: - - port: ${test_port} - protocol: TCP - targetPort: ${test_port} -... diff --git a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/service-account.yaml b/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/service-account.yaml deleted file mode 100644 index 35d99dfae51..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/kubernetes-manifests/service-account.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ${service_account_name} - namespace: ${namespace_name} - labels: - owner: xds-k8s-interop-test - annotations: - iam.gke.io/gcp-service-account: ${gcp_service_account} -... diff --git a/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt b/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt deleted file mode 100644 index 8b5991cffb6..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt +++ /dev/null @@ -1,6 +0,0 @@ --r requirements.lock -# Mirrors black version set in -# https://github.com/grpc/grpc/blob/master/tools/distrib/black_code.sh -black[d]==23.3.0 -isort~=5.9 -# TODO(https://github.com/grpc/grpc/pull/25872): mypy diff --git a/tools/run_tests/xds_k8s_test_driver/requirements.lock b/tools/run_tests/xds_k8s_test_driver/requirements.lock deleted file mode 100644 index 8971d076e03..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/requirements.lock +++ /dev/null @@ -1,41 +0,0 @@ -Mako==1.2.4 -PyYAML==6.0 -absl-py==0.15.0 -google-api-python-client==1.12.11 -google-cloud-secret-manager==2.15.1 -grpcio==1.57.0 -grpcio-health-checking==1.57.0 -grpcio-tools==1.57.0 -grpcio-channelz==1.57.0 -kubernetes==27.2.0 -six==1.16.0 -tenacity==6.3.1 -packaging==23.1 -Pygments==2.14.0 -python-dateutil==2.8.2 -protobuf==4.24.1 -xds-protos==1.58.0rc1 -## The following requirements were added by pip freeze: -cachetools==5.3.1 -certifi==2023.7.22 -charset-normalizer==3.2.0 -google-api-core==2.11.1 -google-auth==2.22.0 -google-auth-httplib2==0.1.0 -googleapis-common-protos==1.60.0 -grpc-google-iam-v1==0.12.6 -grpcio-status==1.57.0 -httplib2==0.22.0 -idna==3.4 -MarkupSafe==2.1.3 -oauthlib==3.2.2 -proto-plus==1.22.3 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pyparsing==3.1.1 -requests==2.31.0 -requests-oauthlib==1.3.1 -rsa==4.9 -uritemplate==3.0.1 -urllib3==1.26.16 -websocket-client==1.6.2 diff --git a/tools/run_tests/xds_k8s_test_driver/requirements.txt b/tools/run_tests/xds_k8s_test_driver/requirements.txt deleted file mode 100644 index ca67bdc911c..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -Mako~=1.1 -PyYAML~=6.0 -absl-py~=0.11 -google-api-python-client~=1.12 -google-cloud-secret-manager~=2.1 -grpcio~=1.57 -grpcio-health-checking~=1.57 -grpcio-tools~=1.57 -grpcio-channelz~=1.57 -kubernetes~=27.2 -six~=1.13 -tenacity~=6.2 -packaging~=23.1 -Pygments~=2.9 -python-dateutil~=2.8 -protobuf~=4.24 -xds-protos==1.58.0rc1 diff --git a/tools/run_tests/xds_k8s_test_driver/run.sh b/tools/run_tests/xds_k8s_test_driver/run.sh deleted file mode 100755 index 0680ce19a6b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/run.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -XDS_K8S_DRIVER_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -readonly XDS_K8S_DRIVER_DIR -readonly XDS_K8S_CONFIG="${XDS_K8S_CONFIG:-$XDS_K8S_DRIVER_DIR/config/local-dev.cfg}" - -display_usage() { - cat </dev/stderr -Convenience script to execute tests/ and helper bin/ scripts. - -USAGE: $0 script_path [arguments] - script_path: path to python script to execute, relative to driver root folder - arguments ...: arguments passed to program in sys.argv - -ENVIRONMENT: - XDS_K8S_CONFIG: file path to the config flagfile, relative to - driver root folder. Default: config/local-dev.cfg - Will be appended as --flagfile="config_absolute_path" argument - XDS_K8S_DRIVER_VENV_DIR: the path to python virtual environment directory - Default: $XDS_K8S_DRIVER_DIR/venv -DESCRIPTION: -This tool performs the following: -1) Ensures python virtual env installed and activated -2) Exports test driver root in PYTHONPATH -3) Automatically appends --flagfile="\$XDS_K8S_CONFIG" argument - -EXAMPLES: -$0 bin/run_td_setup.py --help # list script-specific options -$0 bin/run_td_setup.py --helpfull # list all available options -XDS_K8S_CONFIG=./path-to-flagfile.cfg ./run.sh bin/run_td_setup.py --resource_suffix=override-suffix -$0 tests/baseline_test.py -$0 tests/security_test.py --verbosity=1 --logger_levels=__main__:DEBUG,framework:DEBUG -$0 tests/security_test.py SecurityTest.test_mtls --nocheck_local_certs -EOF - exit 1 -} - -if [[ "$#" -eq 0 || "$1" = "-h" || "$1" = "--help" ]]; then - display_usage -fi - -# Relative paths not yet supported by shellcheck. -# shellcheck source=/dev/null -source "${XDS_K8S_DRIVER_DIR}/bin/ensure_venv.sh" - -cd "${XDS_K8S_DRIVER_DIR}" -export PYTHONPATH="${XDS_K8S_DRIVER_DIR}" -# Split path to python file from the rest of the args. -readonly PY_FILE="$1" -shift - -if [[ "${PY_FILE}" =~ tests/unit($|/) ]]; then - # Do not set the flagfile when running unit tests. - exec python "${PY_FILE}" "$@" -else - # Append args after --flagfile, so they take higher priority. - exec python "${PY_FILE}" --flagfile="${XDS_K8S_CONFIG}" "$@" -fi diff --git a/tools/run_tests/xds_k8s_test_driver/tests/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/__init__.py deleted file mode 100644 index 1c0a3a36a39..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py b/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py deleted file mode 100644 index 3ee07026b76..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/affinity_test.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import time -from typing import List - -from absl import flags -from absl.testing import absltest -from google.protobuf import json_format - -from framework import xds_k8s_flags -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.rpc import grpc_channelz - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_ChannelzChannelState = grpc_channelz.ChannelState -_Lang = skips.Lang - -# Testing consts -_TEST_AFFINITY_METADATA_KEY = "xds_md" -_TD_PROPAGATE_CHECK_INTERVAL_SEC = 10 -_TD_PROPAGATE_TIMEOUT = 600 -_REPLICA_COUNT = 3 -_RPC_COUNT = 100 - - -class AffinityTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - # Force the python client to use the reference server image (Java) - # because the python server doesn't yet support set_not_serving RPC. - # TODO(https://github.com/grpc/grpc/issues/30635): Remove when resolved. - if cls.lang_spec.client_lang == _Lang.PYTHON: - cls.server_image = xds_k8s_flags.SERVER_IMAGE_CANONICAL.value - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang in _Lang.CPP | _Lang.JAVA: - return config.version_gte("v1.40.x") - elif config.client_lang == _Lang.GO: - return config.version_gte("v1.41.x") - elif config.client_lang == _Lang.PYTHON: - # TODO(https://github.com/grpc/grpc/issues/27430): supported after - # the issue is fixed. - return False - elif config.client_lang == _Lang.NODE: - return config.version_gte("v1.10.x") - return True - - def test_affinity(self) -> None: # pylint: disable=too-many-statements - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service( - affinity_header=_TEST_AFFINITY_METADATA_KEY - ) - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient( - test_servers[0], - rpc="EmptyCall", - metadata="EmptyCall:%s:123" % _TEST_AFFINITY_METADATA_KEY, - ) - # Validate the number of received endpoints and affinity configs. - config = test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - self.assertIsNotNone(config) - json_config = json_format.MessageToDict(config) - parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) - logging.info("Client received CSDS response: %s", parsed) - self.assertLen(parsed.endpoints, _REPLICA_COUNT) - self.assertEqual( - parsed.rds["virtualHosts"][0]["routes"][0]["route"][ - "hashPolicy" - ][0]["header"]["headerName"], - _TEST_AFFINITY_METADATA_KEY, - ) - self.assertEqual(parsed.cds[0]["lbPolicy"], "RING_HASH") - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - with self.subTest("10_first_100_affinity_rpcs_pick_same_backend"): - rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT) - json_lb_stats = json_format.MessageToDict(rpc_stats) - rpc_distribution = xds_url_map_testcase.RpcDistributionStats( - json_lb_stats - ) - self.assertEqual(1, rpc_distribution.num_peers) - - # Check subchannel states. - # One should be READY. - ready_channels = test_client.find_subchannels_with_state( - _ChannelzChannelState.READY - ) - self.assertLen( - ready_channels, - 1, - msg=( - "(AffinityTest) The client expected to have one READY" - " subchannel to one of the test servers. Found" - f" {len(ready_channels)} instead." - ), - ) - - # Any remaining subchannels may be in any state. - - # Remember the backend inuse, and turn it down later. - first_backend_inuse = list( - rpc_distribution.raw["rpcsByPeer"].keys() - )[0] - - with self.subTest("11_turn_down_server_in_use"): - for server in test_servers: - if server.hostname == first_backend_inuse: - server.set_not_serving() - - with self.subTest("12_wait_for_unhealth_status_propagation"): - deadline = time.time() + _TD_PROPAGATE_TIMEOUT - parsed = None - try: - while time.time() < deadline: - config = test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - self.assertIsNotNone(config) - json_config = json_format.MessageToDict(config) - parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) - if len(parsed.endpoints) == _REPLICA_COUNT - 1: - break - logging.info( - ( - "CSDS got unexpected endpoints, will retry after %d" - " seconds" - ), - _TD_PROPAGATE_CHECK_INTERVAL_SEC, - ) - time.sleep(_TD_PROPAGATE_CHECK_INTERVAL_SEC) - else: - self.fail( - "unhealthy status did not propagate after 600 seconds" - ) - finally: - logging.info("Client received CSDS response: %s", parsed) - - with self.subTest("12_next_100_affinity_rpcs_pick_different_backend"): - rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT) - json_lb_stats = json_format.MessageToDict(rpc_stats) - rpc_distribution = xds_url_map_testcase.RpcDistributionStats( - json_lb_stats - ) - self.assertEqual(1, rpc_distribution.num_peers) - new_backend_inuse = list(rpc_distribution.raw["rpcsByPeer"].keys())[ - 0 - ] - self.assertNotEqual(new_backend_inuse, first_backend_inuse) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/api_listener_test.py b/tools/run_tests/xds_k8s_test_driver/tests/api_listener_test.py deleted file mode 100644 index 039bf10d1e2..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/api_listener_test.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest -from google.protobuf import json_format - -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.helpers import skips - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -_Lang = skips.Lang - -_TD_CONFIG_RETRY_WAIT_SEC = 2 - - -class ApiListenerTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.PYTHON: - # gRPC Python versions prior to v1.43.x don't support handling empty - # RDS update. - return config.version_gte("v1.43.x") - return True - - def test_api_listener(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service() - - with self.subTest("02_create_default_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_default_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_default_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - test_server: _XdsTestServer - with self.subTest("05_start_test_server"): - test_server = self.startTestServers()[0] - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(test_server) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_server_received_rpcs"): - self.assertSuccessfulRpcs(test_client) - - with self.subTest("10_create_alternate_url_map"): - self.td.create_alternative_url_map( - self.server_xds_host, - self.server_xds_port, - self.td.backend_service, - ) - - # Create alternate target proxy pointing to alternate url_map with the same - # host name in host rule. The port is fixed because they point to the same backend service. - # Therefore we have to choose a non-`0.0.0.0` ip because ip:port needs to be unique. - # We also have to set validate_for_proxyless=false because requires `0.0.0.0` ip. - # See https://github.com/grpc/grpc-java/issues/8009 - with self.subTest("11_create_alternate_target_proxy"): - self.td.create_alternative_target_proxy() - - # Create a second suite of map+tp+fr with the same host name in host rule. - # We set fr ip_address to be different from `0.0.0.0` and then set - # validate_for_proxyless=false because ip:port needs to be unique. - with self.subTest("12_create_alternate_forwarding_rule"): - self.td.create_alternative_forwarding_rule( - self.server_xds_port, ip_address="10.10.10.10" - ) - - with self.subTest("13_test_server_received_rpcs_with_two_url_maps"): - self.assertSuccessfulRpcs(test_client) - raw_config = test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - dumped_config = _DumpedXdsConfig( - json_format.MessageToDict(raw_config) - ) - previous_route_config_version = dumped_config.rds_version - logger.info( - ( - "received client config from CSDS with two url maps, " - "dump config: %s, rds version: %s" - ), - dumped_config, - previous_route_config_version, - ) - - with self.subTest("14_delete_one_url_map_target_proxy_forwarding_rule"): - self.td.delete_forwarding_rule() - self.td.delete_target_grpc_proxy() - self.td.delete_url_map() - - with self.subTest("15_test_server_continues_to_receive_rpcs"): - self.assertRouteConfigUpdateTrafficHandoff( - test_client, - previous_route_config_version, - _TD_CONFIG_RETRY_WAIT_SEC, - xds_k8s_testcase._TD_CONFIG_MAX_WAIT_SEC, - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/app_net_ssa_test.py b/tools/run_tests/xds_k8s_test_driver/tests/app_net_ssa_test.py deleted file mode 100644 index e2a60677baa..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/app_net_ssa_test.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List, Optional - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.rpc import grpc_testing -from framework.test_app import client_app -from framework.test_cases import session_affinity_util - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall - -_REPLICA_COUNT = 3 - -# This is here temporarily to run this test separately from other app_net tests. -# TODO(sergiitk): Move into app_net_test.py - - -class AppNetSsaTest(xds_k8s_testcase.AppNetXdsKubernetesTestCase): - def getClientRpcStats( - self, - test_client: _XdsTestClient, - num_rpcs: int, - *, - metadata_keys: Optional[tuple[str, ...]] = None, - ) -> grpc_testing.LoadBalancerStatsResponse: - """Load all metadata_keys by default.""" - return super().getClientRpcStats( - test_client, - num_rpcs, - metadata_keys=metadata_keys or client_app.REQ_LB_STATS_METADATA_ALL, - ) - - def test_session_affinity_policy(self): - test_servers: List[_XdsTestServer] - - with self.subTest("0_create_health_check"): - self.td.create_health_check() - - with self.subTest("1_create_backend_service"): - self.td.create_backend_service() - - with self.subTest("2_create_mesh"): - self.td.create_mesh() - - with self.subTest("3_create_http_route"): - self.td.create_http_route_with_content( - { - "meshes": [self.td.mesh.url], - "hostnames": [ - f"{self.server_xds_host}:{self.server_xds_port}" - ], - "rules": [ - { - "action": { - "destinations": [ - { - "serviceName": self.td.netsvc.resource_full_name( - self.td.backend_service.name, - "backendServices", - ), - "weight": 1, - }, - ], - "statefulSessionAffinity": { - "cookieTtl": "50s", - }, - }, - }, - ], - } - ) - - with self.subTest("4_run_test_server"): - test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) - - with self.subTest("5_setup_server_backends"): - self.setupServerBackends() - - # Default is round robin LB policy. - - with self.subTest("6_start_test_client"): - test_client: _XdsTestClient = self.startTestClient( - test_servers[0], config_mesh=self.td.mesh.name - ) - - with self.subTest("7_send_first_RPC_and_retrieve_cookie"): - ( - cookie, - chosen_server, - ) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( - self, test_client, test_servers - ) - - with self.subTest("8_send_RPCs_with_cookie"): - test_client.update_config.configure( - rpc_types=(RpcTypeUnaryCall,), - metadata=( - ( - RpcTypeUnaryCall, - "cookie", - cookie, - ), - ), - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, [chosen_server], 10 - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py b/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py deleted file mode 100644 index c399cbcb8d8..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/app_net_test.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - - -class AppNetTest(xds_k8s_testcase.AppNetXdsKubernetesTestCase): - def test_ping_pong(self): - with self.subTest("0_create_health_check"): - self.td.create_health_check() - - with self.subTest("1_create_backend_service"): - self.td.create_backend_service() - - with self.subTest("2_create_mesh"): - self.td.create_mesh() - - with self.subTest("3_create_grpc_route"): - self.td.create_grpc_route( - self.server_xds_host, self.server_xds_port - ) - - test_server: _XdsTestServer - with self.subTest("4_start_test_server"): - test_server = self.startTestServers(replica_count=1)[0] - - with self.subTest("5_setup_server_backends"): - self.setupServerBackends() - - test_client: _XdsTestClient - with self.subTest("6_start_test_client"): - test_client = self.startTestClient( - test_server, config_mesh=self.td.mesh.name - ) - - with self.subTest("7_assert_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("8_assert_successful_rpcs"): - self.assertSuccessfulRpcs(test_client) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py b/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py deleted file mode 100644 index e3c3bff896b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/authz_test.py +++ /dev/null @@ -1,396 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import time -from typing import Optional - -from absl import flags -from absl.testing import absltest -import grpc - -from framework import xds_k8s_testcase -from framework.helpers import skips - -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode -_Lang = skips.Lang - -# The client generates QPS even when it is still loading information from xDS. -# Once it finally connects there will be an outpouring of the bufferred RPCs and -# the server needs time to chew through the backlog, especially since it is -# still a new process and so probably interpreted. The server on one run -# processed 225 RPCs a second, so with the client configured for 25 qps this is -# 40 seconds worth of buffering before starting to drain the backlog. -_SETTLE_DURATION = datetime.timedelta(seconds=5) -_SAMPLE_DURATION = datetime.timedelta(seconds=0.5) - - -class AuthzTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase): - RPC_TYPE_CYCLE = { - "UNARY_CALL": "EMPTY_CALL", - "EMPTY_CALL": "UNARY_CALL", - } - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - # Per "Authorization (RBAC)" in - # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md - if config.client_lang in _Lang.CPP | _Lang.PYTHON: - return config.version_gte("v1.47.x") - elif config.client_lang in _Lang.GO | _Lang.JAVA: - return config.version_gte("v1.42.x") - elif config.client_lang == _Lang.NODE: - return False - return True - - def setUp(self): - super().setUp() - self.next_rpc_type: Optional[int] = None - - def authz_rules(self): - return [ - { - "destinations": { - "hosts": [f"*:{self.server_xds_port}"], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "host-wildcard", - }, - }, - }, - { - "destinations": { - "hosts": [f"*:{self.server_xds_port}"], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "header-regex-a+", - }, - }, - }, - { - "destinations": [ - { - "hosts": [ - f"{self.server_xds_host}:{self.server_xds_port}" - ], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "host-match1", - }, - }, - { - "hosts": [ - f"a-not-it.com:{self.server_xds_port}", - f"{self.server_xds_host}:{self.server_xds_port}", - "z-not-it.com:1", - ], - "ports": [1, self.server_port, 65535], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "host-match2", - }, - }, - ], - }, - { - "destinations": { - "hosts": [ - f"not-the-host:{self.server_xds_port}", - "not-the-host", - ], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "never-match-host", - }, - }, - }, - { - "destinations": { - "hosts": [f"*:{self.server_xds_port}"], - "ports": [1], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "never-match-port", - }, - }, - }, - # b/202058316. The wildcard principal is generating invalid config - # { - # "sources": { - # "principals": ["*"], - # }, - # "destinations": { - # "hosts": [f"*:{self.server_xds_port}"], - # "ports": [self.server_port], - # "httpHeaderMatch": { - # "headerName": "test", - # "regexMatch": "principal-present", - # }, - # }, - # }, - { - "sources": [ - { - "principals": [ - f"spiffe://{self.project}.svc.id.goog/not/the/client", - ], - }, - { - "principals": [ - f"spiffe://{self.project}.svc.id.goog/not/the/client", - ( - f"spiffe://{self.project}.svc.id.goog/ns/" - f"{self.client_namespace}/sa/{self.client_name}" - ), - ], - }, - ], - "destinations": { - "hosts": [f"*:{self.server_xds_port}"], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "match-principal", - }, - }, - }, - { - "sources": { - "principals": [ - f"spiffe://{self.project}.svc.id.goog/not/the/client", - ], - }, - "destinations": { - "hosts": [f"*:{self.server_xds_port}"], - "ports": [self.server_port], - "httpHeaderMatch": { - "headerName": "test", - "regexMatch": "never-match-principal", - }, - }, - }, - ] - - def configure_and_assert( - self, - test_client: _XdsTestClient, - test_metadata_val: Optional[str], - status_code: grpc.StatusCode, - ) -> None: - # Swap method type every sub-test to avoid mixing results - rpc_type = self.next_rpc_type - if rpc_type is None: - stats = test_client.get_load_balancer_accumulated_stats() - for t in self.RPC_TYPE_CYCLE: - if not stats.stats_per_method[t].rpcs_started: - rpc_type = t - self.assertIsNotNone(rpc_type, "All RPC types already used") - self.next_rpc_type = self.RPC_TYPE_CYCLE[rpc_type] - - metadata = None - if test_metadata_val is not None: - metadata = ((rpc_type, "test", test_metadata_val),) - test_client.update_config.configure( - rpc_types=[rpc_type], metadata=metadata - ) - # b/228743575 Python has as race. Give us time to fix it. - stray_rpc_limit = 1 if self.lang_spec.client_lang == _Lang.PYTHON else 0 - self.assertRpcStatusCodes( - test_client, - expected_status=status_code, - duration=_SAMPLE_DURATION, - method=rpc_type, - stray_rpc_limit=stray_rpc_limit, - ) - - def test_plaintext_allow(self) -> None: - self.setupTrafficDirectorGrpc() - self.td.create_authz_policy(action="ALLOW", rules=self.authz_rules()) - self.setupSecurityPolicies( - server_tls=False, - server_mtls=False, - client_tls=False, - client_mtls=False, - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - time.sleep(_SETTLE_DURATION.total_seconds()) - - with self.subTest("01_host_wildcard"): - self.configure_and_assert( - test_client, "host-wildcard", grpc.StatusCode.OK - ) - - with self.subTest("02_no_match"): - self.configure_and_assert( - test_client, "no-such-rule", grpc.StatusCode.PERMISSION_DENIED - ) - self.configure_and_assert( - test_client, None, grpc.StatusCode.PERMISSION_DENIED - ) - - with self.subTest("03_header_regex"): - self.configure_and_assert( - test_client, "header-regex-a", grpc.StatusCode.OK - ) - self.configure_and_assert( - test_client, "header-regex-aa", grpc.StatusCode.OK - ) - self.configure_and_assert( - test_client, "header-regex-", grpc.StatusCode.PERMISSION_DENIED - ) - self.configure_and_assert( - test_client, - "header-regex-ab", - grpc.StatusCode.PERMISSION_DENIED, - ) - self.configure_and_assert( - test_client, - "aheader-regex-a", - grpc.StatusCode.PERMISSION_DENIED, - ) - - with self.subTest("04_host_match"): - self.configure_and_assert( - test_client, "host-match1", grpc.StatusCode.OK - ) - self.configure_and_assert( - test_client, "host-match2", grpc.StatusCode.OK - ) - - with self.subTest("05_never_match_host"): - self.configure_and_assert( - test_client, - "never-match-host", - grpc.StatusCode.PERMISSION_DENIED, - ) - - with self.subTest("06_never_match_port"): - self.configure_and_assert( - test_client, - "never-match-port", - grpc.StatusCode.PERMISSION_DENIED, - ) - - # b/202058316 - # with self.subTest('07_principal_present'): - # self.configure_and_assert(test_client, 'principal-present', - # grpc.StatusCode.PERMISSION_DENIED) - - def test_tls_allow(self) -> None: - self.setupTrafficDirectorGrpc() - self.td.create_authz_policy(action="ALLOW", rules=self.authz_rules()) - self.setupSecurityPolicies( - server_tls=True, - server_mtls=False, - client_tls=True, - client_mtls=False, - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - time.sleep(_SETTLE_DURATION.total_seconds()) - - with self.subTest("01_host_wildcard"): - self.configure_and_assert( - test_client, "host-wildcard", grpc.StatusCode.OK - ) - - with self.subTest("02_no_match"): - self.configure_and_assert( - test_client, None, grpc.StatusCode.PERMISSION_DENIED - ) - - # b/202058316 - # with self.subTest('03_principal_present'): - # self.configure_and_assert(test_client, 'principal-present', - # grpc.StatusCode.PERMISSION_DENIED) - - def test_mtls_allow(self) -> None: - self.setupTrafficDirectorGrpc() - self.td.create_authz_policy(action="ALLOW", rules=self.authz_rules()) - self.setupSecurityPolicies( - server_tls=True, server_mtls=True, client_tls=True, client_mtls=True - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - time.sleep(_SETTLE_DURATION.total_seconds()) - - with self.subTest("01_host_wildcard"): - self.configure_and_assert( - test_client, "host-wildcard", grpc.StatusCode.OK - ) - - with self.subTest("02_no_match"): - self.configure_and_assert( - test_client, None, grpc.StatusCode.PERMISSION_DENIED - ) - - # b/202058316 - # with self.subTest('03_principal_present'): - # self.configure_and_assert(test_client, 'principal-present', - # grpc.StatusCode.OK) - - with self.subTest("04_match_principal"): - self.configure_and_assert( - test_client, "match-principal", grpc.StatusCode.OK - ) - - with self.subTest("05_never_match_principal"): - self.configure_and_assert( - test_client, - "never-match-principal", - grpc.StatusCode.PERMISSION_DENIED, - ) - - def test_plaintext_deny(self) -> None: - self.setupTrafficDirectorGrpc() - self.td.create_authz_policy(action="DENY", rules=self.authz_rules()) - self.setupSecurityPolicies( - server_tls=False, - server_mtls=False, - client_tls=False, - client_mtls=False, - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - time.sleep(_SETTLE_DURATION.total_seconds()) - - with self.subTest("01_host_wildcard"): - self.configure_and_assert( - test_client, "host-wildcard", grpc.StatusCode.PERMISSION_DENIED - ) - - with self.subTest("02_no_match"): - self.configure_and_assert(test_client, None, grpc.StatusCode.OK) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py b/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py deleted file mode 100644 index bf1bcf31acc..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/baseline_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - - -class BaselineTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - def test_traffic_director_grpc_setup(self): - with self.subTest("0_create_health_check"): - self.td.create_health_check() - - with self.subTest("1_create_backend_service"): - self.td.create_backend_service() - - with self.subTest("2_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("3_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("4_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - with self.subTest("5_start_test_server"): - test_server: _XdsTestServer = self.startTestServers()[0] - - with self.subTest("6_add_server_backends_to_backend_service"): - self.setupServerBackends() - - with self.subTest("7_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_server) - - with self.subTest("8_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("9_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/bootstrap_generator_test.py b/tools/run_tests/xds_k8s_test_driver/tests/bootstrap_generator_test.py deleted file mode 100644 index ef6ef14e9d3..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/bootstrap_generator_test.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime -import logging -from typing import List - -from absl import flags -from absl.testing import absltest -from absl.testing import parameterized - -from framework import bootstrap_generator_testcase -from framework import xds_k8s_testcase -from framework.helpers import retryers -from framework.test_app.runners.k8s import k8s_xds_client_runner -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -XdsTestServer = xds_k8s_testcase.XdsTestServer -XdsTestClient = xds_k8s_testcase.XdsTestClient -KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner -KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner -_timedelta = datetime.timedelta - - -# Returns a list of bootstrap generator versions to be tested along with their -# image names. -# -# Whenever we release a new version of the bootstrap generator, we need to add a -# corresponding entry here. -# -# TODO: Update bootstrap generator release instructions to add an entry here, -# after the release is published. -def bootstrap_version_testcases() -> List: - return ( - dict( - version="v0.14.0", - image="gcr.io/grpc-testing/td-grpc-bootstrap:d6baaf7b0e0c63054ac4d9bedc09021ff261d599", - ), - dict( - version="v0.13.0", - image="gcr.io/grpc-testing/td-grpc-bootstrap:203db6ce70452996f4183c30dd4c5ecaada168b0", - ), - dict( - version="v0.12.0", - image="gcr.io/grpc-testing/td-grpc-bootstrap:8765051ef3b742bc5cd20f16de078ae7547f2ba2", - ), - dict( - version="v0.11.0", - image="gcr.io/grpc-testing/td-grpc-bootstrap:b96f7a73314668aee83cbf86ab1e40135a0542fc", - ), - # v0.10.0 uses v2 xDS transport protocol by default. TD only supports v3 - # and we can force the bootstrap generator to emit config with v3 - # support by setting the --include-v3-features-experimental flag to - # true. - # - # TODO: Figure out how to pass flags to the bootstrap generator via the - # client and server runners, and uncomment this version. - # ('v0.10.0', 'gcr.io/grpc-testing/td-grpc-bootstrap:66de7ea0e170351c9fae17232b81adbfb3e80ec3'), - ) - - -# TODO: Reuse service account and namespaces for significant improvements in -# running time. -class BootstrapGeneratorClientTest( - bootstrap_generator_testcase.BootstrapGeneratorBaseTest, - parameterized.TestCase, -): - client_runner: KubernetesClientRunner - server_runner: KubernetesServerRunner - test_server: XdsTestServer - - @classmethod - def setUpClass(cls): - """Hook method for setting up class fixture before running tests in - the class. - """ - super().setUpClass() - - # For client tests, we use a single server instance that can be shared - # across all the parameterized clients. And this server runner will use - # the version of the bootstrap generator as configured via the - # --td_bootstrap_image flag. - cls.server_runner = cls.initKubernetesServerRunner() - cls.test_server = cls.startTestServer( - server_runner=cls.server_runner, - port=cls.server_port, - maintenance_port=cls.server_maintenance_port, - xds_host=cls.server_xds_host, - xds_port=cls.server_xds_port, - ) - - # Load backends. - ( - neg_name, - neg_zones, - ) = cls.server_runner.k8s_namespace.parse_service_neg_status( - cls.server_runner.service_name, cls.server_port - ) - - # Add backends to the Backend Service. - cls.td.backend_service_add_neg_backends(neg_name, neg_zones) - cls.td.wait_for_backends_healthy_status() - - @classmethod - def tearDownClass(cls): - # Remove backends from the Backend Service before closing the server - # runner. - ( - neg_name, - neg_zones, - ) = cls.server_runner.k8s_namespace.parse_service_neg_status( - cls.server_runner.service_name, cls.server_port - ) - cls.td.backend_service_remove_neg_backends(neg_name, neg_zones) - cls.server_runner.cleanup(force=cls.force_cleanup) - super().tearDownClass() - - def tearDown(self): - logger.info("----- TestMethod %s teardown -----", self.id()) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=10), - attempts=3, - log_level=logging.INFO, - ) - try: - retryer(self._cleanup) - except retryers.RetryError: - logger.exception("Got error during teardown") - super().tearDown() - - def _cleanup(self): - self.client_runner.cleanup(force=self.force_cleanup) - - @parameterized.parameters( - (t["version"], t["image"]) for t in bootstrap_version_testcases() - ) - def test_baseline_in_client_with_bootstrap_version(self, version, image): - """Runs the baseline test for multiple versions of the bootstrap - generator on the client. - """ - logger.info( - "----- testing bootstrap generator version %s -----", version - ) - self.client_runner = self.initKubernetesClientRunner( - td_bootstrap_image=image - ) - test_client: XdsTestClient = self.startTestClient(self.test_server) - self.assertXdsConfigExists(test_client) - self.assertSuccessfulRpcs(test_client) - - -# TODO: Use unique client and server deployment names while creating the -# corresponding runners, by suffixing the version of the bootstrap generator -# being tested. Then, run these in parallel. -class BootstrapGeneratorServerTest( - bootstrap_generator_testcase.BootstrapGeneratorBaseTest, - parameterized.TestCase, -): - client_runner: KubernetesClientRunner - server_runner: KubernetesServerRunner - test_server: XdsTestServer - - def tearDown(self): - logger.info("----- TestMethod %s teardown -----", self.id()) - retryer = retryers.constant_retryer( - wait_fixed=_timedelta(seconds=10), - attempts=3, - log_level=logging.INFO, - ) - try: - retryer(self._cleanup) - except retryers.RetryError: - logger.exception("Got error during teardown") - super().tearDown() - - def _cleanup(self): - self.client_runner.cleanup(force=self.force_cleanup) - self.removeServerBackends() - self.server_runner.cleanup(force=self.force_cleanup) - - @parameterized.parameters( - (t["version"], t["image"]) for t in bootstrap_version_testcases() - ) - def test_baseline_in_server_with_bootstrap_version(self, version, image): - """Runs the baseline test for multiple versions of the bootstrap - generator on the server. - """ - logger.info( - "----- Testing bootstrap generator version %s -----", version - ) - self.server_runner = self.initKubernetesServerRunner( - td_bootstrap_image=image - ) - self.test_server = self.startTestServer( - server_runner=self.server_runner, - port=self.server_port, - maintenance_port=self.server_maintenance_port, - xds_host=self.server_xds_host, - xds_port=self.server_xds_port, - bootstrap_version=version, - ) - - # Load backends. - ( - neg_name, - neg_zones, - ) = self.server_runner.k8s_namespace.parse_service_neg_status( - self.server_runner.service_name, self.server_port - ) - - # Add backends to the Backend Service. - self.td.backend_service_add_neg_backends(neg_name, neg_zones) - self.td.wait_for_backends_healthy_status() - - self.client_runner = self.initKubernetesClientRunner() - test_client: XdsTestClient = self.startTestClient(self.test_server) - self.assertXdsConfigExists(test_client) - self.assertSuccessfulRpcs(test_client) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/change_backend_service_test.py b/tools/run_tests/xds_k8s_test_driver/tests/change_backend_service_test.py deleted file mode 100644 index 85dba2582cd..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/change_backend_service_test.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase -from framework.infrastructure import k8s -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - - -class ChangeBackendServiceTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - def setUp(self): - super().setUp() - self.alternate_k8s_namespace = k8s.KubernetesNamespace( - self.k8s_api_manager, self.server_namespace - ) - self.alternate_server_runner = _KubernetesServerRunner( - self.alternate_k8s_namespace, - deployment_name=self.server_name + "-alt", - image_name=self.server_image, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - reuse_namespace=True, - ) - - def cleanup(self): - super().cleanup() - if hasattr(self, "alternate_server_runner"): - self.alternate_server_runner.cleanup( - force=self.force_cleanup, force_namespace=self.force_cleanup - ) - - def test_change_backend_service(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service() - self.td.create_alternative_backend_service() - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - default_test_servers: List[_XdsTestServer] - same_zone_test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - default_test_servers = self.startTestServers() - same_zone_test_servers = self.startTestServers( - server_runner=self.alternate_server_runner - ) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - # Add backend to alternative backend service - ( - neg_name_alt, - neg_zones_alt, - ) = self.alternate_k8s_namespace.parse_service_neg_status( - self.alternate_server_runner.service_name, self.server_port - ) - self.td.alternative_backend_service_add_neg_backends( - neg_name_alt, neg_zones_alt - ) - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(default_test_servers[0]) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - with self.subTest("10_change_backend_service"): - self.td.patch_url_map( - self.server_xds_host, - self.server_xds_port, - self.td.alternative_backend_service, - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, same_zone_test_servers - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/custom_lb_test.py b/tools/run_tests/xds_k8s_test_driver/tests/custom_lb_test.py deleted file mode 100644 index 0e94eb3e819..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/custom_lb_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime -import logging - -from absl import flags -from absl.testing import absltest -import grpc - -from framework import xds_k8s_flags -from framework import xds_k8s_testcase -from framework.helpers import skips - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_Lang = skips.Lang - -_EXPECTED_STATUS = grpc.StatusCode.DATA_LOSS - - -class CustomLbTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - @classmethod - def setUpClass(cls): - """Force the java test server for languages not yet supporting - the `rpc-behavior` feature. - https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#server - """ - super().setUpClass() - # gRPC Java implemented server "error-code-" rpc-behavior in v1.47.x. - # gRPC CPP implemented rpc-behavior in the same version, as custom_lb. - if cls.lang_spec.client_lang in _Lang.JAVA | _Lang.CPP: - return - - # gRPC go, python and node fallback to the gRPC Java. - # TODO(https://github.com/grpc/grpc-go/issues/6288): use go server. - # TODO(https://github.com/grpc/grpc/issues/33134): use python server. - cls.server_image = xds_k8s_flags.SERVER_IMAGE_CANONICAL.value - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.JAVA: - return config.version_gte("v1.47.x") - if config.client_lang == _Lang.CPP: - return config.version_gte("v1.55.x") - if config.client_lang == _Lang.GO: - return config.version_gte("v1.56.x") - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.10.x") - return False - - def test_custom_lb_config(self): - with self.subTest("0_create_health_check"): - self.td.create_health_check() - - # Configures a custom, test LB on the client to instruct the servers - # to always respond with a specific error code. - # - # The first policy in the list is a non-existent one to verify that - # the gRPC client can gracefully move down the list to the valid one - # once it determines the first one is not available. - with self.subTest("1_create_backend_service"): - self.td.create_backend_service( - locality_lb_policies=[ - { - "customPolicy": { - "name": "test.ThisLoadBalancerDoesNotExist", - "data": '{ "foo": "bar" }', - }, - }, - { - "customPolicy": { - "name": "test.RpcBehaviorLoadBalancer", - "data": ( - '{ "rpcBehavior":' - f' "error-code-{_EXPECTED_STATUS.value[0]}" }}' - ), - } - }, - ] - ) - - with self.subTest("2_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("3_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("4_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - with self.subTest("5_start_test_server"): - test_server: _XdsTestServer = self.startTestServers()[0] - - with self.subTest("6_add_server_backends_to_backend_service"): - self.setupServerBackends() - - with self.subTest("7_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_server) - - with self.subTest("8_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - # Verify status codes from the servers have the configured one. - with self.subTest("9_test_server_returned_configured_status_code"): - self.assertRpcStatusCodes( - test_client, - expected_status=_EXPECTED_STATUS, - duration=datetime.timedelta(seconds=10), - method="UNARY_CALL", - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/failover_test.py b/tools/run_tests/xds_k8s_test_driver/tests/failover_test.py deleted file mode 100644 index 4895a6c5a35..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/failover_test.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_flags -from framework import xds_k8s_testcase -from framework.helpers import skips -from framework.infrastructure import k8s -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) -xds_k8s_flags.require_secondary_context(__file__) - -# Type aliases -_Lang = skips.Lang -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - - -class FailoverTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - REPLICA_COUNT = 3 - MAX_RATE_PER_ENDPOINT = 100 - - @classmethod - def setUpClass(cls): - super().setUpClass() - # Force the python client to use the reference server image (Java) - # because the python server doesn't yet support set_not_serving RPC. - # TODO(https://github.com/grpc/grpc/issues/30635): Remove when resolved. - if cls.lang_spec.client_lang == _Lang.PYTHON: - cls.server_image = xds_k8s_flags.SERVER_IMAGE_CANONICAL.value - - def setUp(self): - super().setUp() - self.secondary_server_runner = _KubernetesServerRunner( - k8s.KubernetesNamespace( - self.secondary_k8s_api_manager, self.server_namespace - ), - deployment_name=self.server_name + "-alt", - image_name=self.server_image, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - # This runner's namespace created in the secondary cluster, - # so it's not reused and must be cleaned up. - reuse_namespace=False, - ) - - def cleanup(self): - super().cleanup() - if hasattr(self, "secondary_server_runner"): - self.secondary_server_runner.cleanup( - force=self.force_cleanup, force_namespace=self.force_cleanup - ) - - def test_failover(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service() - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - default_test_servers: List[_XdsTestServer] - alternate_test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - default_test_servers = self.startTestServers( - replica_count=self.REPLICA_COUNT - ) - - alternate_test_servers = self.startTestServers( - server_runner=self.secondary_server_runner - ) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends( - max_rate_per_endpoint=self.MAX_RATE_PER_ENDPOINT - ) - self.setupServerBackends( - server_runner=self.secondary_server_runner, - max_rate_per_endpoint=self.MAX_RATE_PER_ENDPOINT, - ) - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(default_test_servers[0]) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_primary_locality_receives_requests"): - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers - ) - - with self.subTest( - "10_secondary_locality_receives_no_requests_on_partial_primary_failure" - ): - default_test_servers[0].set_not_serving() - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers[1:] - ) - - with self.subTest("11_gentle_failover"): - default_test_servers[1].set_not_serving() - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers[2:] + alternate_test_servers - ) - - with self.subTest( - "12_secondary_locality_receives_requests_on_primary_failure" - ): - default_test_servers[2].set_not_serving() - self.assertRpcsEventuallyGoToGivenServers( - test_client, alternate_test_servers - ) - - with self.subTest("13_traffic_resumes_to_healthy_backends"): - for i in range(self.REPLICA_COUNT): - default_test_servers[i].set_serving() - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/gamma/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/gamma/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/gamma/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/gamma/affinity_test.py b/tools/run_tests/xds_k8s_test_driver/tests/gamma/affinity_test.py deleted file mode 100644 index 156b9b123d1..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/gamma/affinity_test.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List, Optional - -from absl import flags -from absl.testing import absltest - -from framework import xds_gamma_testcase -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.rpc import grpc_testing -from framework.test_app import client_app -from framework.test_app import server_app -from framework.test_cases import session_affinity_util - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -_XdsTestServer = server_app.XdsTestServer -_XdsTestClient = client_app.XdsTestClient -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall - -_REPLICA_COUNT = 3 - -# TODO(rbellevi): set this property on the prestop hook test class -# We never actually hit this timeout under normal circumstances, so this large -# value is acceptable. -# termination_grace_period_seconds: int = 600 - - -class AffinityTest(xds_gamma_testcase.GammaXdsKubernetesTestCase): - def getClientRpcStats( - self, - test_client: _XdsTestClient, - num_rpcs: int, - *, - metadata_keys: Optional[tuple[str, ...]] = None, - ) -> grpc_testing.LoadBalancerStatsResponse: - """Load all metadata_keys by default.""" - return super().getClientRpcStats( - test_client, - num_rpcs, - metadata_keys=metadata_keys or client_app.REQ_LB_STATS_METADATA_ALL, - ) - - def test_session_affinity_filter(self): - test_servers: List[_XdsTestServer] - with self.subTest("01_run_test_server"): - test_servers = self.startTestServers( - replica_count=_REPLICA_COUNT, - route_template="gamma/route_http_ssafilter.yaml", - ) - - with self.subTest("02_create_ssa_filter"): - self.server_runner.createSessionAffinityFilter() - - # Default is round robin LB policy. - - with self.subTest("03_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_servers[0]) - - with self.subTest("04_send_first_RPC_and_retrieve_cookie"): - ( - cookie, - chosen_server, - ) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( - self, test_client, test_servers - ) - - with self.subTest("05_send_RPCs_with_cookie"): - test_client.update_config.configure( - rpc_types=(RpcTypeUnaryCall,), - metadata=( - ( - RpcTypeUnaryCall, - "cookie", - cookie, - ), - ), - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, [chosen_server], 10 - ) - - def test_session_affinity_policy_with_route_target(self): - test_servers: List[_XdsTestServer] - with self.subTest("01_run_test_server"): - test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) - - with self.subTest("02_create_ssa_policy"): - self.server_runner.createSessionAffinityPolicy( - "gamma/session_affinity_policy_route.yaml" - ) - - # Default is round robin LB policy. - - with self.subTest("03_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_servers[0]) - - with self.subTest("04_send_first_RPC_and_retrieve_cookie"): - ( - cookie, - chosen_server, - ) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( - self, test_client, test_servers - ) - - with self.subTest("05_send_RPCs_with_cookie"): - test_client.update_config.configure( - rpc_types=(RpcTypeUnaryCall,), - metadata=( - ( - RpcTypeUnaryCall, - "cookie", - cookie, - ), - ), - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, [chosen_server], 10 - ) - - def test_session_affinity_policy_with_service_target(self): - test_servers: List[_XdsTestServer] - with self.subTest("01_run_test_server"): - test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) - - with self.subTest("02_create_ssa_policy"): - self.server_runner.createSessionAffinityPolicy( - "gamma/session_affinity_policy_service.yaml" - ) - - # Default is round robin LB policy. - - with self.subTest("03_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_servers[0]) - - with self.subTest("04_send_first_RPC_and_retrieve_cookie"): - ( - cookie, - chosen_server, - ) = session_affinity_util.assert_eventually_retrieve_cookie_and_server( - self, test_client, test_servers - ) - - with self.subTest("05_send_RPCs_with_cookie"): - test_client.update_config.configure( - rpc_types=(RpcTypeUnaryCall,), - metadata=( - ( - RpcTypeUnaryCall, - "cookie", - cookie, - ), - ), - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, [chosen_server], 10 - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/gamma/csm_observability_test.py b/tools/run_tests/xds_k8s_test_driver/tests/gamma/csm_observability_test.py deleted file mode 100644 index 7f0c09a13bf..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/gamma/csm_observability_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest - -from framework import xds_gamma_testcase -from framework import xds_k8s_testcase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - - -class CsmObservabilityTest(xds_gamma_testcase.GammaXdsKubernetesTestCase): - def test_ping_pong(self): - # TODO(sergiitk): [GAMMA] Consider moving out custom gamma - # resource creation out of self.startTestServers() - with self.subTest("1_run_test_server"): - test_server: _XdsTestServer = self.startTestServers( - enable_csm_observability=True - )[0] - - with self.subTest("2_start_test_client"): - test_client: _XdsTestClient = self.startTestClient( - test_server, enable_csm_observability=True - ) - - with self.subTest("3_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/gamma/gamma_baseline_test.py b/tools/run_tests/xds_k8s_test_driver/tests/gamma/gamma_baseline_test.py deleted file mode 100644 index 77a33ad745b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/gamma/gamma_baseline_test.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest - -from framework import xds_gamma_testcase -from framework import xds_k8s_testcase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - - -class GammaBaselineTest(xds_gamma_testcase.GammaXdsKubernetesTestCase): - def test_ping_pong(self): - # TODO(sergiitk): [GAMMA] Consider moving out custom gamma - # resource creation out of self.startTestServers() - with self.subTest("1_run_test_server"): - test_server: _XdsTestServer = self.startTestServers()[0] - - with self.subTest("2_start_test_client"): - test_client: _XdsTestClient = self.startTestClient(test_server) - - with self.subTest("3_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py b/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py deleted file mode 100644 index 5713f85e310..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/outlier_detection_test.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_flags -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.helpers import skips - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) -flags.mark_flag_as_required("server_image_canonical") - -# Type aliases -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_Lang = skips.Lang - -# Testing consts -_QPS = 100 -_REPLICA_COUNT = 5 - - -class OutlierDetectionTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - """ - Implementation of https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#outlier_detection - - This test verifies that the client applies the outlier detection - configuration and temporarily drops traffic to a server that fails - requests. - """ - - @classmethod - def setUpClass(cls): - """Force the java test server for languages not yet supporting - the `rpc-behavior` feature. - - https://github.com/grpc/grpc/blob/master/doc/xds-test-descriptions.md#server - """ - super().setUpClass() - # gRPC Java implemented server "error-code-" rpc-behavior in v1.47.x. - if cls.lang_spec.client_lang == _Lang.JAVA: - return - - # gRPC CPP implemented server "hostname" rpc-behavior in v1.57.x, - # see https://github.com/grpc/grpc/pull/33446. - if ( - cls.lang_spec.client_lang == _Lang.CPP - and cls.lang_spec.version_gte("v1.57.x") - ): - return - - # gRPC go, python and node fallback to the gRPC Java. - # TODO(https://github.com/grpc/grpc-go/issues/6288): use go server. - # TODO(https://github.com/grpc/grpc/issues/33134): use python server. - cls.server_image = xds_k8s_flags.SERVER_IMAGE_CANONICAL.value - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang in _Lang.CPP | _Lang.PYTHON: - return config.version_gte("v1.48.x") - if config.client_lang == _Lang.JAVA: - return config.version_gte("v1.49.x") - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.6.x") - if config.client_lang == _Lang.GO: - # TODO(zasweq): Update when the feature makes in a version branch. - return config.version_gte("master") - return False - - def test_outlier_detection(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_service"): - self.td.create_backend_service( - outlier_detection={ - "interval": {"seconds": 2, "nanos": 0}, - "successRateRequestVolume": 20, - } - ) - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - test_servers = self.startTestServers(replica_count=_REPLICA_COUNT) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(test_servers[0], qps=_QPS) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_servers_received_rpcs_from_test_client"): - self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers) - - rpc_types = (RpcTypeUnaryCall,) - with self.subTest("10_chosen_server_removed_by_outlier_detection"): - test_client.update_config.configure( - rpc_types=rpc_types, - metadata=( - ( - RpcTypeUnaryCall, - "rpc-behavior", - f"hostname={test_servers[0].hostname} error-code-2", - ), - ), - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, test_servers[1:] - ) - - with self.subTest("11_ejected_server_returned_after_failures_stopped"): - test_client.update_config.configure(rpc_types=rpc_types) - self.assertRpcsEventuallyGoToGivenServers(test_client, test_servers) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/remove_neg_test.py b/tools/run_tests/xds_k8s_test_driver/tests/remove_neg_test.py deleted file mode 100644 index cd690b187a7..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/remove_neg_test.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase -from framework.infrastructure import k8s -from framework.test_app.runners.k8s import k8s_xds_server_runner - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner - - -class RemoveNegTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - def setUp(self): - super().setUp() - self.alternate_server_runner = _KubernetesServerRunner( - k8s.KubernetesNamespace( - self.k8s_api_manager, self.server_namespace - ), - deployment_name=self.server_name + "-alt", - image_name=self.server_image, - gcp_service_account=self.gcp_service_account, - td_bootstrap_image=self.td_bootstrap_image, - gcp_project=self.project, - gcp_api_manager=self.gcp_api_manager, - xds_server_uri=self.xds_server_uri, - network=self.network, - debug_use_port_forwarding=self.debug_use_port_forwarding, - reuse_namespace=True, - ) - - def cleanup(self): - super().cleanup() - if hasattr(self, "alternate_server_runner"): - self.alternate_server_runner.cleanup( - force=self.force_cleanup, force_namespace=self.force_cleanup - ) - - def test_remove_neg(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service() - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - default_test_servers: List[_XdsTestServer] - same_zone_test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - default_test_servers = self.startTestServers() - same_zone_test_servers = self.startTestServers( - server_runner=self.alternate_server_runner - ) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - self.setupServerBackends(server_runner=self.alternate_server_runner) - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(default_test_servers[0]) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - with self.subTest("10_remove_neg"): - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers + same_zone_test_servers - ) - self.removeServerBackends( - server_runner=self.alternate_server_runner - ) - self.assertRpcsEventuallyGoToGivenServers( - test_client, default_test_servers - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/round_robin_test.py b/tools/run_tests/xds_k8s_test_driver/tests/round_robin_test.py deleted file mode 100644 index 6d3144f065b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/round_robin_test.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import List - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - - -class RoundRobinTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - def test_round_robin(self) -> None: - REPLICA_COUNT = 2 - - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service() - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - test_servers = self.startTestServers(replica_count=REPLICA_COUNT) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - - test_client: _XdsTestClient - with self.subTest("07_start_test_client"): - test_client = self.startTestClient(test_servers[0]) - - with self.subTest("08_test_client_xds_config_exists"): - self.assertXdsConfigExists(test_client) - - with self.subTest("09_test_server_received_rpcs_from_test_client"): - self.assertSuccessfulRpcs(test_client) - - with self.subTest("10_round_robin"): - num_rpcs = 100 - expected_rpcs_per_replica = num_rpcs / REPLICA_COUNT - - rpcs_by_peer = self.getClientRpcStats( - test_client, num_rpcs - ).rpcs_by_peer - total_requests_received = sum(rpcs_by_peer[x] for x in rpcs_by_peer) - self.assertEqual( - total_requests_received, num_rpcs, "Wrong number of RPCS" - ) - for server in test_servers: - hostname = server.hostname - self.assertIn( - hostname, - rpcs_by_peer, - f"Server {hostname} did not receive RPCs", - ) - self.assertLessEqual( - abs(rpcs_by_peer[hostname] - expected_rpcs_per_replica), - 1, - f"Wrong number of RPCs for server {hostname}", - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/security_test.py b/tools/run_tests/xds_k8s_test_driver/tests/security_test.py deleted file mode 100644 index 06364030720..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/security_test.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2020 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -from absl import flags -from absl.testing import absltest - -from framework import xds_k8s_testcase -from framework.helpers import rand -from framework.helpers import skips - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient -_SecurityMode = xds_k8s_testcase.SecurityXdsKubernetesTestCase.SecurityMode -_Lang = skips.Lang - - -class SecurityTest(xds_k8s_testcase.SecurityXdsKubernetesTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang in ( - _Lang.CPP | _Lang.GO | _Lang.JAVA | _Lang.PYTHON - ): - # Versions prior to v1.41.x don't support PSM Security. - # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md - return config.version_gte("v1.41.x") - elif config.client_lang == _Lang.NODE: - return False - return True - - def test_mtls(self): - """mTLS test. - - Both client and server configured to use TLS and mTLS. - """ - self.setupTrafficDirectorGrpc() - self.setupSecurityPolicies( - server_tls=True, server_mtls=True, client_tls=True, client_mtls=True - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - - self.assertTestAppSecurity(_SecurityMode.MTLS, test_client, test_server) - self.assertSuccessfulRpcs(test_client) - logger.info("[SUCCESS] mTLS security mode confirmed.") - - def test_tls(self): - """TLS test. - - Both client and server configured to use TLS and not use mTLS. - """ - self.setupTrafficDirectorGrpc() - self.setupSecurityPolicies( - server_tls=True, - server_mtls=False, - client_tls=True, - client_mtls=False, - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - - self.assertTestAppSecurity(_SecurityMode.TLS, test_client, test_server) - self.assertSuccessfulRpcs(test_client) - logger.info("[SUCCESS] TLS security mode confirmed.") - - def test_plaintext_fallback(self): - """Plain-text fallback test. - - Control plane provides no security config so both client and server - fallback to plaintext based on fallback-credentials. - """ - self.setupTrafficDirectorGrpc() - self.setupSecurityPolicies( - server_tls=False, - server_mtls=False, - client_tls=False, - client_mtls=False, - ) - - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends() - test_client: _XdsTestClient = self.startSecureTestClient(test_server) - - self.assertTestAppSecurity( - _SecurityMode.PLAINTEXT, test_client, test_server - ) - self.assertSuccessfulRpcs(test_client) - logger.info("[SUCCESS] Plaintext security mode confirmed.") - - def test_mtls_error(self): - """Negative test: mTLS Error. - - Server expects client mTLS cert, but client configured only for TLS. - - Note: because this is a negative test we need to make sure the mTLS - failure happens after receiving the correct configuration at the - client. To ensure that we will perform the following steps in that - sequence: - - - Creation of a backendService, and attaching the backend (NEG) - - Creation of the Server mTLS Policy, and attaching to the ECS - - Creation of the Client TLS Policy, and attaching to the backendService - - Creation of the urlMap, targetProxy, and forwardingRule - - With this sequence we are sure that when the client receives the - endpoints of the backendService the security-config would also have - been received as confirmed by the TD team. - """ - # Create backend service - self.td.setup_backend_for_grpc( - health_check_port=self.server_maintenance_port - ) - - # Start server and attach its NEGs to the backend service, but - # until they become healthy. - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends(wait_for_healthy_status=False) - - # Setup policies and attach them. - self.setupSecurityPolicies( - server_tls=True, - server_mtls=True, - client_tls=True, - client_mtls=False, - ) - - # Create the routing rule map. - self.td.setup_routing_rule_map_for_grpc( - self.server_xds_host, self.server_xds_port - ) - # Now that TD setup is complete, Backend Service can be populated - # with healthy backends (NEGs). - self.td.wait_for_backends_healthy_status() - - # Start the client, but don't wait for it to report a healthy channel. - test_client: _XdsTestClient = self.startSecureTestClient( - test_server, wait_for_server_channel_ready=False - ) - - self.assertClientCannotReachServerRepeatedly(test_client) - logger.info( - "[SUCCESS] Client's connectivity state is consistent with a mTLS " - "error caused by not presenting mTLS certificate to the server." - ) - - def test_server_authz_error(self): - """Negative test: AuthZ error. - - Client does not authorize server because of mismatched SAN name. - The order of operations is the same as in `test_mtls_error`. - """ - # Create backend service - self.td.setup_backend_for_grpc( - health_check_port=self.server_maintenance_port - ) - - # Start server and attach its NEGs to the backend service, but - # until they become healthy. - test_server: _XdsTestServer = self.startSecureTestServer() - self.setupServerBackends(wait_for_healthy_status=False) - - # Regular TLS setup, but with client policy configured using - # intentionality incorrect server_namespace. - self.td.setup_server_security( - server_namespace=self.server_namespace, - server_name=self.server_name, - server_port=self.server_port, - tls=True, - mtls=False, - ) - incorrect_namespace = f"incorrect-namespace-{rand.rand_string()}" - self.td.setup_client_security( - server_namespace=incorrect_namespace, - server_name=self.server_name, - tls=True, - mtls=False, - ) - - # Create the routing rule map. - self.td.setup_routing_rule_map_for_grpc( - self.server_xds_host, self.server_xds_port - ) - # Now that TD setup is complete, Backend Service can be populated - # with healthy backends (NEGs). - self.td.wait_for_backends_healthy_status() - - # Start the client, but don't wait for it to report a healthy channel. - test_client: _XdsTestClient = self.startSecureTestClient( - test_server, wait_for_server_channel_ready=False - ) - - self.assertClientCannotReachServerRepeatedly(test_client) - logger.info( - "[SUCCESS] Client's connectivity state is consistent with " - "AuthZ error caused by server presenting incorrect SAN." - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py b/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py deleted file mode 100644 index e3dcae0a3ae..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/subsetting_test.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -from typing import List - -from absl import flags -from absl import logging -from absl.testing import absltest -from google.protobuf import json_format - -from framework import xds_k8s_testcase -from framework import xds_url_map_testcase -from framework.helpers import skips - -flags.adopt_module_key_flags(xds_k8s_testcase) - -# Type aliases -_XdsTestServer = xds_k8s_testcase.XdsTestServer -_XdsTestClient = xds_k8s_testcase.XdsTestClient - -_SUBSET_SIZE = 4 -_NUM_BACKENDS = 8 -_NUM_CLIENTS = 3 - - -class SubsettingTest(xds_k8s_testcase.RegularXdsKubernetesTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - # Subsetting is an experimental feature where most work is done on the - # server-side. We limit it to only run on master branch to save - # resources. - return config.version_gte("master") - - def test_subsetting_basic(self) -> None: - with self.subTest("00_create_health_check"): - self.td.create_health_check() - - with self.subTest("01_create_backend_services"): - self.td.create_backend_service(subset_size=_SUBSET_SIZE) - - with self.subTest("02_create_url_map"): - self.td.create_url_map(self.server_xds_host, self.server_xds_port) - - with self.subTest("03_create_target_proxy"): - self.td.create_target_proxy() - - with self.subTest("04_create_forwarding_rule"): - self.td.create_forwarding_rule(self.server_xds_port) - - test_servers: List[_XdsTestServer] - with self.subTest("05_start_test_servers"): - test_servers = self.startTestServers(replica_count=_NUM_BACKENDS) - - with self.subTest("06_add_server_backends_to_backend_services"): - self.setupServerBackends() - - rpc_distribution = collections.defaultdict(int) - with self.subTest("07_start_test_client"): - for i in range(_NUM_CLIENTS): - # Clean created client pods if there is any. - if self.client_runner.time_start_requested: - # TODO(sergiitk): Speed up by reusing the namespace. - self.client_runner.cleanup() - - # Create a test client - test_client: _XdsTestClient = self.startTestClient( - test_servers[0] - ) - # Validate the number of received endpoints - config = test_client.csds.fetch_client_status( - log_level=logging.INFO - ) - self.assertIsNotNone(config) - json_config = json_format.MessageToDict(config) - parsed = xds_url_map_testcase.DumpedXdsConfig(json_config) - logging.info( - "Client %d received endpoints (len=%s): %s", - i, - len(parsed.endpoints), - parsed.endpoints, - ) - self.assertLen(parsed.endpoints, _SUBSET_SIZE) - # Record RPC stats - lb_stats = self.getClientRpcStats( - test_client, _NUM_BACKENDS * 25 - ) - for key, value in lb_stats.rpcs_by_peer.items(): - rpc_distribution[key] += value - - with self.subTest("08_log_rpc_distribution"): - server_entries = sorted( - rpc_distribution.items(), key=lambda x: -x[1] - ) - # Validate if clients are receiving different sets of backends (3 - # client received a total of 4 unique backends == FAIL, a total of 5 - # unique backends == PASS) - self.assertGreater(len(server_entries), _SUBSET_SIZE) - logging.info( - "RPC distribution (len=%s): %s", - len(server_entries), - server_entries, - ) - peak = server_entries[0][1] - mean = sum(map(lambda x: x[1], server_entries)) / len( - server_entries - ) - logging.info( - "Peak=%d Mean=%.1f Peak-to-Mean-Ratio=%.2f", - peak, - mean, - peak / mean, - ) - - -if __name__ == "__main__": - absltest.main(failfast=True) diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/__main__.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/__main__.py deleted file mode 100644 index 0ca1df94f82..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/__main__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2023 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Discover and run all unit tests recursively.""" - -import pathlib - -from absl.testing import absltest - - -def load_tests(loader: absltest.TestLoader, unused_tests, unused_pattern): - unit_tests_root = pathlib.Path(__file__).parent - return loader.discover(f"{unit_tests_root}", pattern="*_test.py") - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/skips_test.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/skips_test.py deleted file mode 100644 index 7c297c8bdb3..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/helpers/skips_test.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Optional - -from absl.testing import absltest -from absl.testing import parameterized -from packaging import version as pkg_version - -from framework.helpers import skips - - -class TestConfigVersionGteTest(parameterized.TestCase): - """ - Unit test for the version comparison helper TestConfig.version_gte. - - See TestConfig.version_gte for doc for the description of the special cases - mentioned below. - """ - - INVALID_VERSIONS: tuple[str, ...] = ( - "1.2.xx", - "1.x.x", - "alpha", - "1.x.alpha", - "1.2.x.alpha", - "1.x.3", - "x.1.3", - "v1.x.3", - "v.1.2.0", - "y1.2.0", - ) - - VALID_VERSIONS: tuple[str, ...] = ( - "1.2.x", - "1.2.0", - "v1.2.x", - "v1.2.0", - "V1.2.0", - "1.999.x", - "2.999.x", - ) - - VALID_DEV_VERSIONS: tuple[str, ...] = ( - "dev-1.3.x", - "dev-1.3.0", - "dev-master", - ) - - @classmethod - def _make_test_config(cls, version: str) -> skips.TestConfig: - return skips.TestConfig( - client_lang=skips.Lang.UNKNOWN, - server_lang=skips.Lang.UNKNOWN, - version=version, - ) - - @classmethod - def _v_prefix_combos(cls, left: str, right: str) -> tuple[str, str]: - yield left, right - yield f"v{left}", right - yield left, f"v{right}" - yield f"v{left}", f"v{right}" - - @parameterized.parameters( - # Greater than. - ("1.3.0", "1.2.0", True), - ("1.3.1", "1.3.0", True), - ("2.0.0", "1.999.999", True), - ("2.0", "1.999.999", True), - # Equal. - ("1.3.0", "1.3.0", True), - ("1.3.0", "1.3", True), - ("1.3.1", "1.3.1", True), - # Less than. - ("1.2.0", "1.3.0", False), - ("1.2.0", "1.2.1", False), - ("1.999.999", "2.0.0", False), - ("1.999.999", "2.0", False), - ) - def test_basic_parse_and_compare( - self, - version_left: str, - version_right: str, - result: bool, - ): - """Verifies basic parsing and comparison.""" - self.assertEqual( - self._make_test_config(version_left).version_gte(version_right), - result, - msg=f"Expected ({version_left} >= {version_right}) == {result}", - ) - - @parameterized.parameters( - ("1.3.x", "1.2.x", True), # Greater than. - ("1.3.x", "1.3.x", True), # Equal. - ("1.3.x", "1.4.x", False), # Less than. - ) - def test_v_prefix( - self, - version_left: str, - version_right: str, - result: bool, - ): - """Verifies the "v" prefix is allowed.""" - for left, right in self._v_prefix_combos(version_left, version_right): - self.assertEqual( - self._make_test_config(left).version_gte(right), - result, - msg=f"Expected ({left} >= {right}) == {result}", - ) - - @parameterized.parameters( - # Greater than. - ("1.3.x", "1.2.9", True), - ("1.3.1", "1.3.x", True), - ("1.3.x", "1.2.x", True), - ("2.0.x", "1.999.x", True), - # Equal. - ("1.3.x", "1.3.x", True), - ("1.3.0", "1.3.x", True), - ("1.3.x", "1.3.0", True), - # Less than - ("1.3.x", "1.3.1", False), - ("1.3.x", "1.4.x", False), - ("1.3.x", "1.4.0", False), - ) - def test_x_in_version_patch( - self, - version_left: str, - version_right: str, - result: bool, - ): - """Verifies the patch-level "x" is equivalent to patch-level "0".""" - self.assertEqual( - self._make_test_config(version_left).version_gte(version_right), - result, - msg=f"Expected ({version_left} >= {version_right}) == {result}", - ) - - @parameterized.product( - version_left=("master", "dev", "dev-master", None), - version_right=("master", "dev", "dev-master", "1.999.x", "dev-1.999.x"), - ) - def test_special_case_left_always_gte( - self, - version_left: str, - version_right: str, - ): - """ - Special case 1: Versions "master", "dev", "dev-master" on the left - are always "greater or equal" than any version on the right. - """ - self.assertTrue( - self._make_test_config(version_left).version_gte(version_right), - msg=f"Expected {version_left} >= {version_right}", - ) - - @parameterized.parameters( - ("dev-1.3.x", "1.2.x", True), - ("dev-1.3.x", "1.3.x", True), - ("dev-1.3.x", "1.4.x", False), - ("dev-1.3.x", "master", False), - ) - def test_special_case_left_dev_prefix_ignored( - self, - version_left: str, - version_right: str, - result: bool, - ): - """ - Special case 2: Versions "dev-VERSION" on the left behave the same - as the VERSION on the right. - """ - self.assertEqual( - self._make_test_config(version_left).version_gte(version_right), - result, - msg=f"Expected ({version_left} >= {version_right}) == {result}", - ) - - @parameterized.parameters( - *INVALID_VERSIONS, - *VALID_DEV_VERSIONS, - ) - def test_right_invalid_version(self, version_right: Optional[str]): - """ - Verifies that invalid versions on the right raise InvalidVersion. - - Note that special case 2, versions "dev-VERSION", are not allowed - on the right. - """ - test_config = self._make_test_config("1.42.x") - with self.assertRaises(pkg_version.InvalidVersion): - test_config.version_gte(version_right) - - @parameterized.parameters(*INVALID_VERSIONS) - def test_left_invalid_version(self, version_left: Optional[str]): - """ - Verifies that invalid versions on the left raise InvalidVersion. - """ - test_config = self._make_test_config(version_left) - with self.assertRaises(pkg_version.InvalidVersion): - test_config.version_gte("1.42.x") - - @parameterized.parameters( - *VALID_VERSIONS, - *VALID_DEV_VERSIONS, - None, - ) - def test_left_valid_version(self, version_left: Optional[str]): - """ - Verifies that valid versions on the left don't raise InvalidVersion. - """ - test_config = self._make_test_config(version_left) - try: - test_config.version_gte("1.42.x") - except pkg_version.InvalidVersion as e: - self.fail( - f"Version on the left {version_left} expected to be valid" - ) - - @parameterized.parameters(*VALID_VERSIONS) - def test_right_valid_version(self, version_right: Optional[str]): - """ - Verifies that valid versions on the right don't raise InvalidVersion. - """ - test_config = self._make_test_config("1.42.x") - try: - test_config.version_gte(version_right) - except pkg_version.InvalidVersion as e: - self.fail( - f"Version on the right {version_right} expected to be valid" - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/__init__.py deleted file mode 100644 index d921d237a32..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/client_app_test.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/client_app_test.py deleted file mode 100644 index a309e5d5391..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/client_app_test.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Optional - -from absl.testing import absltest - -from framework.test_app import client_app - -# Alias -XdsTestClient = client_app.XdsTestClient - -# Test values. -CANNED_IP: str = "10.0.0.42" -CANNED_RPC_PORT: int = 1111 -CANNED_HOSTNAME: str = "test-client.local" -CANNED_SERVER_TARGET: str = "xds:///test-server" - - -class ClientAppTest(absltest.TestCase): - """Unit test for the ClientApp.""" - - def test_constructor(self): - xds_client = XdsTestClient( - ip=CANNED_IP, - rpc_port=CANNED_RPC_PORT, - hostname=CANNED_HOSTNAME, - server_target=CANNED_SERVER_TARGET, - ) - # Channels list empty. - self.assertEmpty(xds_client.channels) - - # Test fields set as is. - self.assertEqual(xds_client.ip, CANNED_IP) - self.assertEqual(xds_client.rpc_port, CANNED_RPC_PORT) - self.assertEqual(xds_client.server_target, CANNED_SERVER_TARGET) - self.assertEqual(xds_client.hostname, CANNED_HOSTNAME) - - # Test optional argument defaults. - self.assertEqual(xds_client.rpc_host, CANNED_IP) - self.assertEqual(xds_client.maintenance_port, CANNED_RPC_PORT) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/server_app_test.py b/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/server_app_test.py deleted file mode 100644 index b43acff2c9c..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/unit/test_app/server_app_test.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2023 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Optional - -from absl.testing import absltest - -from framework.test_app import server_app - -# Alias -XdsTestServer = server_app.XdsTestServer - -# Test values. -CANNED_IP: str = "10.0.0.43" -CANNED_RPC_PORT: int = 2222 -CANNED_HOSTNAME: str = "test-server.local" -CANNED_XDS_HOST: str = "xds-test-server" -CANNED_XDS_PORT: int = 42 - - -class ServerAppTest(absltest.TestCase): - """Unit test for the XdsTestServer.""" - - def test_constructor(self): - xds_server = XdsTestServer( - ip=CANNED_IP, - rpc_port=CANNED_RPC_PORT, - hostname=CANNED_HOSTNAME, - ) - # Channels list empty. - self.assertEmpty(xds_server.channels) - - # Test fields set as is. - self.assertEqual(xds_server.ip, CANNED_IP) - self.assertEqual(xds_server.rpc_port, CANNED_RPC_PORT) - self.assertEqual(xds_server.hostname, CANNED_HOSTNAME) - - # Test optional argument defaults. - self.assertEqual(xds_server.rpc_host, CANNED_IP) - self.assertEqual(xds_server.maintenance_port, CANNED_RPC_PORT) - self.assertEqual(xds_server.secure_mode, False) - - def test_xds_address(self): - """Verifies the behavior of set_xds_address(), xds_address, xds_uri.""" - xds_server = XdsTestServer( - ip=CANNED_IP, - rpc_port=CANNED_RPC_PORT, - hostname=CANNED_HOSTNAME, - ) - self.assertEqual(xds_server.xds_uri, "", msg="Must be empty when unset") - - xds_server.set_xds_address(CANNED_XDS_HOST, CANNED_XDS_PORT) - self.assertEqual(xds_server.xds_uri, "xds:///xds-test-server:42") - - xds_server.set_xds_address(CANNED_XDS_HOST, None) - self.assertEqual( - xds_server.xds_uri, - "xds:///xds-test-server", - msg="Must not contain ':port' when the port is not set", - ) - - xds_server.set_xds_address(None, None) - self.assertEqual(xds_server.xds_uri, "", msg="Must be empty when reset") - - xds_server.set_xds_address(None, CANNED_XDS_PORT) - self.assertEqual( - xds_server.xds_uri, - "", - msg="Must be empty when only port is set", - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/__init__.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/__init__.py deleted file mode 100644 index 712a2e1de2a..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/__main__.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/__main__.py deleted file mode 100644 index ff8c4855c2c..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/__main__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from absl.testing import absltest - -from framework import xds_url_map_testcase # Needed for xDS flags - -_TEST_CASE_FOLDER = os.path.dirname(__file__) - - -def load_tests(loader: absltest.TestLoader, unused_tests, unused_pattern): - return loader.discover( - _TEST_CASE_FOLDER, - pattern="*" + xds_url_map_testcase.URL_MAP_TESTCASE_FILE_SUFFIX, - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py deleted file mode 100644 index a3ccfac1318..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/affinity_test.py +++ /dev/null @@ -1,269 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.infrastructure import traffic_director -from framework.rpc import grpc_channelz -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -_NUM_RPCS = 150 -_TEST_METADATA_KEY = traffic_director.TEST_AFFINITY_METADATA_KEY -_TEST_METADATA_VALUE_UNARY = "unary_yranu" -_TEST_METADATA_VALUE_EMPTY = "empty_ytpme" -_TEST_METADATA_NUMERIC_KEY = "xds_md_numeric" -_TEST_METADATA_NUMERIC_VALUE = "159" - -_TEST_METADATA = ( - (RpcTypeUnaryCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_UNARY), - (RpcTypeEmptyCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_EMPTY), - ( - RpcTypeUnaryCall, - _TEST_METADATA_NUMERIC_KEY, - _TEST_METADATA_NUMERIC_VALUE, - ), -) - -_ChannelzChannelState = grpc_channelz.ChannelState - - -def _is_supported(config: skips.TestConfig) -> bool: - # Per "Ring hash" in - # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md - if config.client_lang in _Lang.CPP | _Lang.JAVA: - return config.version_gte("v1.40.x") - elif config.client_lang == _Lang.GO: - return config.version_gte("v1.41.x") - elif config.client_lang == _Lang.PYTHON: - # TODO(https://github.com/grpc/grpc/issues/27430): supported after - # the issue is fixed. - return False - elif config.client_lang == _Lang.NODE: - return False - return True - - -class TestHeaderBasedAffinity(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def client_init_config(rpc: str, metadata: str): - # Config the init RPCs to send with the same set of metadata. Without - # this, the init RPCs will not have headers, and will pick random - # backends (behavior of RING_HASH). This is necessary to only one - # sub-channel is picked and used from the beginning, thus the channel - # will only create this one sub-channel. - return "EmptyCall", "EmptyCall:%s:%s" % ( - _TEST_METADATA_KEY, - _TEST_METADATA_VALUE_EMPTY, - ) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - # 3 endpoints in the affinity backend service. - self.assertNumEndpoints(xds_config, 3) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "hashPolicy" - ][0]["header"]["headerName"], - _TEST_METADATA_KEY, - ) - self.assertEqual(xds_config.cds[0]["lbPolicy"], "RING_HASH") - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - # Only one backend should receive traffic, even though there are 3 - # backends. - self.assertEqual(1, rpc_distribution.num_peers) - self.assertLen( - test_client.find_subchannels_with_state( - _ChannelzChannelState.READY - ), - 1, - ) - - # Any remaining subchannels may be in any state. - - # Send 150 RPCs without headers. RPCs without headers will pick random - # backends. After this, we expect to see all backends to be connected. - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall], - num_rpcs=_NUM_RPCS, - ) - self.assertEqual(3, rpc_distribution.num_peers) - self.assertLen( - test_client.find_subchannels_with_state( - _ChannelzChannelState.READY - ), - 3, - ) - - -class TestHeaderBasedAffinityMultipleHeaders( - xds_url_map_testcase.XdsUrlMapTestCase -): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def client_init_config(rpc: str, metadata: str): - # Config the init RPCs to send with the same set of metadata. Without - # this, the init RPCs will not have headers, and will pick random - # backends (behavior of RING_HASH). This is necessary to only one - # sub-channel is picked and used from the beginning, thus the channel - # will only create this one sub-channel. - return "EmptyCall", "EmptyCall:%s:%s" % ( - _TEST_METADATA_KEY, - _TEST_METADATA_VALUE_EMPTY, - ) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - # Update default service to the affinity service. - path_matcher[ - "defaultService" - ] = GcpResourceManager().affinity_backend_service() - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - # 3 endpoints in the affinity backend service. - self.assertNumEndpoints(xds_config, 3) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "hashPolicy" - ][0]["header"]["headerName"], - _TEST_METADATA_KEY, - ) - self.assertEqual(xds_config.cds[0]["lbPolicy"], "RING_HASH") - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - # Only one backend should receive traffic, even though there are 3 - # backends. - self.assertEqual(1, rpc_distribution.num_peers) - self.assertLen( - test_client.find_subchannels_with_state( - _ChannelzChannelState.READY - ), - 1, - ) - - # Any remaining subchannels may be in any state. - - empty_call_peer = list( - rpc_distribution.raw["rpcsByMethod"]["EmptyCall"][ - "rpcsByPeer" - ].keys() - )[0] - # Send RPCs with a different metadata value, try different values to - # verify that the client will pick a different backend. - # - # EmptyCalls will be sent with the same metadata as before, and - # UnaryCalls will be sent with headers from ["0".."29"]. We check the - # endpoint picked for UnaryCall, and stop as soon as one different from - # the EmptyCall peer is picked. - # - # Note that there's a small chance all the headers would still pick the - # same backend used by EmptyCall. But there will be over a thousand - # nodes on the ring (default min size is 1024), and the probability of - # picking the same backend should be fairly small. - different_peer_picked = False - for i in range(30): - new_metadata = ( - ( - RpcTypeEmptyCall, - _TEST_METADATA_KEY, - _TEST_METADATA_VALUE_EMPTY, - ), - (RpcTypeUnaryCall, _TEST_METADATA_KEY, str(i)), - ) - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall, RpcTypeUnaryCall], - metadata=new_metadata, - num_rpcs=_NUM_RPCS, - ) - unary_call_peer = list( - rpc_distribution.raw["rpcsByMethod"]["UnaryCall"][ - "rpcsByPeer" - ].keys() - )[0] - if unary_call_peer != empty_call_peer: - different_peer_picked = True - break - self.assertTrue( - different_peer_picked, - ( - "the same endpoint was picked for all the headers, expect a " - "different endpoint to be picked" - ), - ) - self.assertLen( - test_client.find_subchannels_with_state( - _ChannelzChannelState.READY - ), - 2, - ) - # Any remaining subchannels may be in any state. - - -# TODO: add more test cases -# 1. based on the basic test, turn down the backend in use, then verify that all -# RPCs are sent to another backend - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/csds_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/csds_test.py deleted file mode 100644 index 98bfac57c3b..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/csds_test.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -_NUM_RPCS = 50 - - -class TestBasicCsds(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.5.x") - return True - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - # Validate Endpoint Configs - self.assertNumEndpoints(xds_config, 1) - # Validate Node - self.assertEqual( - self.test_client.ip, xds_config["node"]["metadata"]["INSTANCE_IP"] - ) - # Validate Listeners - self.assertIsNotNone(xds_config.lds) - self.assertEqual(self.hostname(), xds_config.lds["name"]) - # Validate Route Configs - self.assertTrue(xds_config.rds["virtualHosts"]) - # Validate Clusters - self.assertEqual(1, len(xds_config.cds)) - self.assertEqual("EDS", xds_config.cds[0]["type"]) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeUnaryCall, RpcTypeEmptyCall], - num_rpcs=_NUM_RPCS, - ) - self.assertEqual(_NUM_RPCS, rpc_distribution.num_oks) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/fault_injection_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/fault_injection_test.py deleted file mode 100644 index 11d0a364767..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/fault_injection_test.py +++ /dev/null @@ -1,459 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import time -from typing import Tuple - -from absl import flags -from absl.testing import absltest -import grpc - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient -ExpectedResult = xds_url_map_testcase.ExpectedResult -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -# The first batch of RPCs don't count towards the result of test case. They are -# meant to prove the communication between driver and client is fine. -_NUM_RPCS = 10 -_LENGTH_OF_RPC_SENDING_SEC = 16 -# We are using sleep to synchronize test driver and the client... Even though -# the client is sending at QPS rate, we can't assert that exactly QPS * -# SLEEP_DURATION number of RPC is finished. The final completed RPC might be -# slightly more or less. -_NON_RANDOM_ERROR_TOLERANCE = 0.01 -# For random generator involved test cases, we want to be more loose about the -# final result. Otherwise, we will need more test duration (sleep duration) and -# more accurate communication mechanism. The accurate of random number -# generation is not the intention of this test. -_ERROR_TOLERANCE = 0.2 -_DELAY_CASE_APPLICATION_TIMEOUT_SEC = 1 -_BACKLOG_WAIT_TIME_SEC = 20 - - -def _build_fault_injection_route_rule( - abort_percentage: int = 0, delay_percentage: int = 0 -): - return { - "priority": 0, - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/UnaryCall"} - ], - "service": GcpResourceManager().default_backend_service(), - "routeAction": { - "faultInjectionPolicy": { - "abort": { - "httpStatus": 401, - "percentage": abort_percentage, - }, - "delay": { - "fixedDelay": {"seconds": "20"}, - "percentage": delay_percentage, - }, - } - }, - } - - -def _wait_until_backlog_cleared( - test_client: XdsTestClient, timeout: int = _BACKLOG_WAIT_TIME_SEC -): - """Wait until the completed RPC is close to started RPC. - - For delay injected test cases, there might be a backlog of RPCs due to slow - initialization of the client. E.g., if initialization took 20s and qps is - 25, then there will be a backlog of 500 RPCs. In normal test cases, this is - fine, because RPCs will fail immediately. But for delay injected test cases, - the RPC might linger much longer and affect the stability of test results. - """ - logger.info("Waiting for RPC backlog to clear for %d seconds", timeout) - deadline = time.time() + timeout - while time.time() < deadline: - stats = test_client.get_load_balancer_accumulated_stats() - ok = True - for rpc_type in [RpcTypeUnaryCall, RpcTypeEmptyCall]: - started = stats.num_rpcs_started_by_method.get(rpc_type, 0) - completed = stats.num_rpcs_succeeded_by_method.get( - rpc_type, 0 - ) + stats.num_rpcs_failed_by_method.get(rpc_type, 0) - # We consider the backlog is healthy, if the diff between started - # RPCs and completed RPCs is less than 1.5 QPS. - if abs(started - completed) > xds_url_map_testcase.QPS.value * 1.1: - logger.info( - "RPC backlog exist: rpc_type=%s started=%s completed=%s", - rpc_type, - started, - completed, - ) - time.sleep(_DELAY_CASE_APPLICATION_TIMEOUT_SEC) - ok = False - else: - logger.info( - "RPC backlog clear: rpc_type=%s started=%s completed=%s", - rpc_type, - started, - completed, - ) - if ok: - # Both backlog of both types of RPCs is clear, success, return. - return - - raise RuntimeError("failed to clear RPC backlog in %s seconds" % timeout) - - -def _is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.4.x") - return True - - -class TestZeroPercentFaultInjection(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=0, delay_percentage=0 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual("20s", filter_config["delay"]["fixedDelay"]) - self.assertEqual( - 0, filter_config["delay"]["percentage"].get("numerator", 0) - ) - self.assertEqual( - "MILLION", filter_config["delay"]["percentage"]["denominator"] - ) - self.assertEqual(401, filter_config["abort"]["httpStatus"]) - self.assertEqual( - 0, filter_config["abort"]["percentage"].get("numerator", 0) - ) - self.assertEqual( - "MILLION", filter_config["abort"]["percentage"]["denominator"] - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.OK, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -class TestNonMatchingFaultInjection(xds_url_map_testcase.XdsUrlMapTestCase): - """EMPTY_CALL is not fault injected, so it should succeed.""" - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def client_init_config(rpc: str, metadata: str): - # Python interop client will stuck if the traffic is slow (in this case, - # 20s injected). The purpose of this test is examining the un-injected - # traffic is not impacted, so it's fine to just send un-injected - # traffic. - return "EmptyCall", metadata - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=100, delay_percentage=100 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - # The first route rule for UNARY_CALL is fault injected - self.assertEqual( - "/grpc.testing.TestService/UnaryCall", - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["path"], - ) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual("20s", filter_config["delay"]["fixedDelay"]) - self.assertEqual( - 1000000, filter_config["delay"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["delay"]["percentage"]["denominator"] - ) - self.assertEqual(401, filter_config["abort"]["httpStatus"]) - self.assertEqual( - 1000000, filter_config["abort"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["abort"]["percentage"]["denominator"] - ) - # The second route rule for all other RPCs is untouched - self.assertNotIn( - "envoy.filters.http.fault", - xds_config.rds["virtualHosts"][0]["routes"][1].get( - "typedPerFilterConfig", {} - ), - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeEmptyCall, - status_code=grpc.StatusCode.OK, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -@absltest.skip("20% RPC might pass immediately, reason unknown") -class TestAlwaysDelay(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=0, delay_percentage=100 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual("20s", filter_config["delay"]["fixedDelay"]) - self.assertEqual( - 1000000, filter_config["delay"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["delay"]["percentage"]["denominator"] - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - num_rpcs=_NUM_RPCS, - app_timeout=_DELAY_CASE_APPLICATION_TIMEOUT_SEC, - ) - _wait_until_backlog_cleared(test_client) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.DEADLINE_EXCEEDED, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -class TestAlwaysAbort(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=100, delay_percentage=0 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual(401, filter_config["abort"]["httpStatus"]) - self.assertEqual( - 1000000, filter_config["abort"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["abort"]["percentage"]["denominator"] - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.UNAUTHENTICATED, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -class TestDelayHalf(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=0, delay_percentage=50 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual("20s", filter_config["delay"]["fixedDelay"]) - self.assertEqual( - 500000, filter_config["delay"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["delay"]["percentage"]["denominator"] - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - num_rpcs=_NUM_RPCS, - app_timeout=_DELAY_CASE_APPLICATION_TIMEOUT_SEC, - ) - _wait_until_backlog_cleared(test_client) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.DEADLINE_EXCEEDED, - ratio=0.5, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_ERROR_TOLERANCE, - ) - - -class TestAbortHalf(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_fault_injection_route_rule( - abort_percentage=50, delay_percentage=0 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - filter_config = xds_config.rds["virtualHosts"][0]["routes"][0][ - "typedPerFilterConfig" - ]["envoy.filters.http.fault"] - self.assertEqual(401, filter_config["abort"]["httpStatus"]) - self.assertEqual( - 500000, filter_config["abort"]["percentage"]["numerator"] - ) - self.assertEqual( - "MILLION", filter_config["abort"]["percentage"]["denominator"] - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.UNAUTHENTICATED, - ratio=0.5, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_ERROR_TOLERANCE, - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/header_matching_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/header_matching_test.py deleted file mode 100644 index 1f559d735e1..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/header_matching_test.py +++ /dev/null @@ -1,488 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -_NUM_RPCS = 150 -_TEST_METADATA_KEY = "xds_md" -_TEST_METADATA_VALUE_UNARY = "unary_yranu" -_TEST_METADATA_VALUE_EMPTY = "empty_ytpme" -_TEST_METADATA_NUMERIC_KEY = "xds_md_numeric" -_TEST_METADATA_NUMERIC_VALUE = "159" - -_TEST_METADATA = ( - (RpcTypeUnaryCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_UNARY), - (RpcTypeEmptyCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_EMPTY), - ( - RpcTypeUnaryCall, - _TEST_METADATA_NUMERIC_KEY, - _TEST_METADATA_NUMERIC_VALUE, - ), -) - - -def _is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.3.x") - return True - - -class TestExactMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header ExactMatch -> alternate_backend_service. - # EmptyCall is sent with the metadata. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "exactMatch": _TEST_METADATA_VALUE_EMPTY, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["exactMatch"], - _TEST_METADATA_VALUE_EMPTY, - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -@absltest.skip("the xDS config is good, but distribution is wrong.") -class TestPrefixMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header PrefixMatch -> alternate_backend_service. - # UnaryCall is sent with the metadata. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "prefixMatch": _TEST_METADATA_VALUE_UNARY[:2], - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["prefixMatch"], - _TEST_METADATA_VALUE_UNARY[:2], - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestSuffixMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header SuffixMatch -> alternate_backend_service. - # EmptyCall is sent with the metadata. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "suffixMatch": _TEST_METADATA_VALUE_EMPTY[-2:], - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["suffixMatch"], - _TEST_METADATA_VALUE_EMPTY[-2:], - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -class TestPresentMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header 'xds_md_numeric' present -> alternate_backend_service. - # UnaryCall is sent with the metadata, so will be sent to alternative. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_NUMERIC_KEY, - "presentMatch": True, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_NUMERIC_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["presentMatch"], - True, - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestInvertMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header invert ExactMatch -> alternate_backend_service. - # UnaryCall is sent with the metadata, so will be sent to - # default. EmptyCall will be sent to alternative. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "exactMatch": _TEST_METADATA_VALUE_UNARY, - "invertMatch": True, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["invertMatch"], - True, - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeUnaryCall, RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual(_NUM_RPCS, rpc_distribution.num_oks) - self.assertEqual( - 0, rpc_distribution.unary_call_alternative_service_rpc_count - ) - self.assertEqual( - 0, rpc_distribution.empty_call_default_service_rpc_count - ) - - -class TestRangeMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Header 'xds_md_numeric' range [100,200] -> alternate_backend_service. - # UnaryCall is sent with the metadata in range. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_NUMERIC_KEY, - "rangeMatch": { - "rangeStart": "100", - "rangeEnd": "200", - }, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_NUMERIC_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["rangeMatch"]["start"], - "100", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["rangeMatch"]["end"], - "200", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeUnaryCall, RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual(_NUM_RPCS, rpc_distribution.num_oks) - self.assertEqual( - 0, rpc_distribution.unary_call_default_service_rpc_count - ) - self.assertEqual( - 0, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -class TestRegexMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = ( - [ - { - "priority": 0, - # Header RegexMatch -> alternate_backend_service. - # EmptyCall is sent with the metadata. - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "regexMatch": "^%s.*%s$" - % ( - _TEST_METADATA_VALUE_EMPTY[:2], - _TEST_METADATA_VALUE_EMPTY[-2:], - ), - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ], - ) - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["safeRegexMatch"]["regex"], - "^%s.*%s$" - % (_TEST_METADATA_VALUE_EMPTY[:2], _TEST_METADATA_VALUE_EMPTY[-2:]), - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/metadata_filter_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/metadata_filter_test.py deleted file mode 100644 index eb131d626b0..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/metadata_filter_test.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2022 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest - -from framework import xds_url_map_testcase -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -_NUM_RPCS = 150 -_TEST_METADATA_KEY = "xds_md" -_TEST_METADATA_VALUE_EMPTY = "empty_ytpme" -_TEST_METADATA = ( - (RpcTypeEmptyCall, _TEST_METADATA_KEY, _TEST_METADATA_VALUE_EMPTY), -) -match_labels = [ - {"name": "TRAFFICDIRECTOR_NETWORK_NAME", "value": "default-vpc"} -] -not_match_labels = [{"name": "fake", "value": "fail"}] - - -class TestMetadataFilterMatchAll(xds_url_map_testcase.XdsUrlMapTestCase): - """ " The test url-map has two routeRules: the higher priority routes to - the default backends, but is supposed to be filtered out by TD because - of non-matching metadata filters. The lower priority routes to alternative - backends and metadata filter matches. Thus, it verifies that TD evaluates - metadata filters correctly.""" - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - "matchRules": [ - { - "prefixMatch": "/", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ALL", - "filterLabels": not_match_labels, - } - ], - } - ], - "service": GcpResourceManager().default_backend_service(), - }, - { - "priority": 1, - "matchRules": [ - { - "prefixMatch": "/grpc.testing.TestService/Empty", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "exactMatch": _TEST_METADATA_VALUE_EMPTY, - } - ], - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ALL", - "filterLabels": match_labels, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - }, - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual(len(xds_config.rds["virtualHosts"][0]["routes"]), 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["prefix"], - "/grpc.testing.TestService/Empty", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["exactMatch"], - _TEST_METADATA_VALUE_EMPTY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][1]["match"]["prefix"], - "", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -class TestMetadataFilterMatchAny(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - "matchRules": [ - { - "prefixMatch": "/", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ANY", - "filterLabels": not_match_labels, - } - ], - } - ], - "service": GcpResourceManager().default_backend_service(), - }, - { - "priority": 1, - "matchRules": [ - { - "prefixMatch": "/grpc.testing.TestService/Unary", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ANY", - "filterLabels": not_match_labels + match_labels, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - }, - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual(len(xds_config.rds["virtualHosts"][0]["routes"]), 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["prefix"], - "/grpc.testing.TestService/Unary", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][1]["match"]["prefix"], - "", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestMetadataFilterMatchAnyAndAll(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - "matchRules": [ - { - "prefixMatch": "/", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ALL", - "filterLabels": not_match_labels + match_labels, - } - ], - } - ], - "service": GcpResourceManager().default_backend_service(), - }, - { - "priority": 1, - "matchRules": [ - { - "prefixMatch": "/grpc.testing.TestService/Unary", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ANY", - "filterLabels": not_match_labels + match_labels, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - }, - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual(len(xds_config.rds["virtualHosts"][0]["routes"]), 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["prefix"], - "/grpc.testing.TestService/Unary", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][1]["match"]["prefix"], - "", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestMetadataFilterMatchMultipleRules( - xds_url_map_testcase.XdsUrlMapTestCase -): - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - "matchRules": [ - { - "prefixMatch": "/", - "headerMatches": [ - { - "headerName": _TEST_METADATA_KEY, - "exactMatch": _TEST_METADATA_VALUE_EMPTY, - } - ], - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ANY", - "filterLabels": match_labels, - } - ], - } - ], - "service": GcpResourceManager().alternative_backend_service(), - }, - { - "priority": 1, - "matchRules": [ - { - "prefixMatch": "/grpc.testing.TestService/Unary", - "metadataFilters": [ - { - "filterMatchCriteria": "MATCH_ALL", - "filterLabels": match_labels, - } - ], - } - ], - "service": GcpResourceManager().default_backend_service(), - }, - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual(len(xds_config.rds["virtualHosts"][0]["routes"]), 3) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["name"], - _TEST_METADATA_KEY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["headers"][ - 0 - ]["exactMatch"], - _TEST_METADATA_VALUE_EMPTY, - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][1]["match"]["prefix"], - "/grpc.testing.TestService/Unary", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][2]["match"]["prefix"], - "", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeEmptyCall], - metadata=_TEST_METADATA, - num_rpcs=_NUM_RPCS, - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/path_matching_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/path_matching_test.py deleted file mode 100644 index 3d92859a4de..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/path_matching_test.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -XdsTestClient = client_app.XdsTestClient -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -_NUM_RPCS = 150 - - -def _is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == _Lang.NODE: - return config.version_gte("v1.3.x") - return True - - -class TestFullPathMatchEmptyCall(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # FullPath EmptyCall -> alternate_backend_service. - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/EmptyCall"} - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["path"], - "/grpc.testing.TestService/EmptyCall", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=[RpcTypeEmptyCall], num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -class TestFullPathMatchUnaryCall(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # FullPath EmptyCall -> alternate_backend_service. - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/UnaryCall"} - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["path"], - "/grpc.testing.TestService/UnaryCall", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestTwoRoutesAndPrefixMatch(xds_url_map_testcase.XdsUrlMapTestCase): - """This test case is similar to the one above (but with route services - swapped). This test has two routes (full_path and the default) to match - EmptyCall, and both routes set alternative_backend_service as the action. - This forces the client to handle duplicate Clusters in the RDS response.""" - - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Prefix UnaryCall -> default_backend_service. - "matchRules": [ - {"prefixMatch": "/grpc.testing.TestService/Unary"} - ], - "service": GcpResourceManager().default_backend_service(), - }, - { - "priority": 1, - # FullPath EmptyCall -> alternate_backend_service. - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/EmptyCall"} - ], - "service": GcpResourceManager().alternative_backend_service(), - }, - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["prefix"], - "/grpc.testing.TestService/Unary", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][1]["match"]["path"], - "/grpc.testing.TestService/EmptyCall", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, - rpc_types=[RpcTypeUnaryCall, RpcTypeEmptyCall], - num_rpcs=_NUM_RPCS, - ) - self.assertEqual(0, rpc_distribution.num_failures) - self.assertEqual( - 0, rpc_distribution.unary_call_alternative_service_rpc_count - ) - self.assertEqual( - 0, rpc_distribution.empty_call_default_service_rpc_count - ) - - -class TestRegexMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # Regex UnaryCall -> alternate_backend_service. - "matchRules": [ - { - "regexMatch": ( # Unary methods with any services. - r"^\/.*\/UnaryCall$" - ) - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"][ - "safeRegex" - ]["regex"], - r"^\/.*\/UnaryCall$", - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=(RpcTypeUnaryCall,), num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.unary_call_alternative_service_rpc_count - ) - - -class TestCaseInsensitiveMatch(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - # ignoreCase EmptyCall -> alternate_backend_service. - "matchRules": [ - { - # Case insensitive matching. - "fullPathMatch": "/gRpC.tEsTinG.tEstseRvice/empTycaLl", - "ignoreCase": True, - } - ], - "service": GcpResourceManager().alternative_backend_service(), - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 2) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"]["path"], - "/gRpC.tEsTinG.tEstseRvice/empTycaLl", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["match"][ - "caseSensitive" - ], - False, - ) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - rpc_distribution = self.configure_and_send( - test_client, rpc_types=[RpcTypeEmptyCall], num_rpcs=_NUM_RPCS - ) - self.assertEqual( - _NUM_RPCS, rpc_distribution.empty_call_alternative_service_rpc_count - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py deleted file mode 100644 index 74574bde6c0..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/retry_test.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple - -from absl import flags -from absl.testing import absltest -import grpc - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -XdsTestClient = client_app.XdsTestClient -ExpectedResult = xds_url_map_testcase.ExpectedResult -_Lang = skips.Lang - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -# The first batch of RPCs don't count towards the result of test case. They are -# meant to prove the communication between driver and client is fine. -_NUM_RPCS = 10 -_LENGTH_OF_RPC_SENDING_SEC = 16 -# We are using sleep to synchronize test driver and the client... Even though -# the client is sending at QPS rate, we can't assert that exactly QPS * -# SLEEP_DURATION number of RPC is finished. The final completed RPC might be -# slightly more or less. -_NON_RANDOM_ERROR_TOLERANCE = 0.01 -_RPC_BEHAVIOR_HEADER_NAME = "rpc-behavior" - - -def _build_retry_route_rule(retryConditions, num_retries): - return { - "priority": 0, - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/UnaryCall"} - ], - "service": GcpResourceManager().default_backend_service(), - "routeAction": { - "retryPolicy": { - "retryConditions": retryConditions, - "numRetries": num_retries, - } - }, - } - - -def _is_supported(config: skips.TestConfig) -> bool: - # Per "Retry" in - # https://github.com/grpc/grpc/blob/master/doc/grpc_xds_features.md - if config.client_lang in _Lang.CPP | _Lang.JAVA | _Lang.PYTHON: - return config.version_gte("v1.40.x") - elif config.client_lang == _Lang.GO: - return config.version_gte("v1.41.x") - elif config.client_lang == _Lang.NODE: - return config.version_gte("v1.8.x") - return True - - -class TestRetryUpTo3AttemptsAndFail(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_retry_route_rule( - retryConditions=["unavailable"], num_retries=3 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - retry_config = xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "retryPolicy" - ] - self.assertEqual(3, retry_config["numRetries"]) - self.assertEqual("unavailable", retry_config["retryOn"]) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - metadata=[ - ( - RpcTypeUnaryCall, - _RPC_BEHAVIOR_HEADER_NAME, - "succeed-on-retry-attempt-4,error-code-14", - ) - ], - num_rpcs=_NUM_RPCS, - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.UNAVAILABLE, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -class TestRetryUpTo4AttemptsAndSucceed(xds_url_map_testcase.XdsUrlMapTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - return _is_supported(config) - - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - _build_retry_route_rule( - retryConditions=["unavailable"], num_retries=4 - ) - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - retry_config = xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "retryPolicy" - ] - self.assertEqual(4, retry_config["numRetries"]) - self.assertEqual("unavailable", retry_config["retryOn"]) - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - metadata=[ - ( - RpcTypeUnaryCall, - _RPC_BEHAVIOR_HEADER_NAME, - "succeed-on-retry-attempt-4,error-code-14", - ) - ], - num_rpcs=_NUM_RPCS, - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.OK, - ratio=1, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_NON_RANDOM_ERROR_TOLERANCE, - ) - - -if __name__ == "__main__": - absltest.main() diff --git a/tools/run_tests/xds_k8s_test_driver/tests/url_map/timeout_test.py b/tools/run_tests/xds_k8s_test_driver/tests/url_map/timeout_test.py deleted file mode 100644 index 32f3f3a35ae..00000000000 --- a/tools/run_tests/xds_k8s_test_driver/tests/url_map/timeout_test.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2021 The gRPC Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from typing import Tuple -import unittest - -from absl import flags -from absl.testing import absltest -import grpc - -from framework import xds_url_map_testcase -from framework.helpers import skips -from framework.test_app import client_app - -# Type aliases -HostRule = xds_url_map_testcase.HostRule -PathMatcher = xds_url_map_testcase.PathMatcher -GcpResourceManager = xds_url_map_testcase.GcpResourceManager -DumpedXdsConfig = xds_url_map_testcase.DumpedXdsConfig -RpcTypeUnaryCall = xds_url_map_testcase.RpcTypeUnaryCall -RpcTypeEmptyCall = xds_url_map_testcase.RpcTypeEmptyCall -ExpectedResult = xds_url_map_testcase.ExpectedResult -XdsTestClient = client_app.XdsTestClient -XdsUrlMapTestCase = xds_url_map_testcase.XdsUrlMapTestCase - -logger = logging.getLogger(__name__) -flags.adopt_module_key_flags(xds_url_map_testcase) - -# The first batch of RPCs don't count towards the result of test case. They are -# meant to prove the communication between driver and client is fine. -_NUM_RPCS = 25 -_LENGTH_OF_RPC_SENDING_SEC = 10 -_ERROR_TOLERANCE = 0.1 - - -class _BaseXdsTimeOutTestCase(XdsUrlMapTestCase): - @staticmethod - def url_map_change( - host_rule: HostRule, path_matcher: PathMatcher - ) -> Tuple[HostRule, PathMatcher]: - path_matcher["routeRules"] = [ - { - "priority": 0, - "matchRules": [ - {"fullPathMatch": "/grpc.testing.TestService/UnaryCall"} - ], - "service": GcpResourceManager().default_backend_service(), - "routeAction": { - "maxStreamDuration": { - "seconds": 3, - }, - }, - } - ] - return host_rule, path_matcher - - def xds_config_validate(self, xds_config: DumpedXdsConfig): - self.assertNumEndpoints(xds_config, 1) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "maxStreamDuration" - ]["maxStreamDuration"], - "3s", - ) - self.assertEqual( - xds_config.rds["virtualHosts"][0]["routes"][0]["route"][ - "maxStreamDuration" - ]["grpcTimeoutHeaderMax"], - "3s", - ) - - def rpc_distribution_validate(self, unused_test_client): - raise NotImplementedError() - - -class TestTimeoutInRouteRule(_BaseXdsTimeOutTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - # TODO(lidiz) either add support for rpc-behavior to other languages, or we - # should always use Java server as backend. - if config.server_lang != skips.Lang.JAVA: - return False - if config.client_lang == skips.Lang.NODE: - return config.version_gte("v1.4.x") - return True - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=[RpcTypeUnaryCall, RpcTypeEmptyCall], - # UnaryCall and EmptyCall both sleep-4. - # UnaryCall timeouts, EmptyCall succeeds. - metadata=( - (RpcTypeUnaryCall, "rpc-behavior", "sleep-4"), - (RpcTypeEmptyCall, "rpc-behavior", "sleep-4"), - ), - num_rpcs=_NUM_RPCS, - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.DEADLINE_EXCEEDED, - ), - ExpectedResult( - rpc_type=RpcTypeEmptyCall, status_code=grpc.StatusCode.OK - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_ERROR_TOLERANCE, - ) - - -class TestTimeoutInApplication(_BaseXdsTimeOutTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - # TODO(lidiz) either add support for rpc-behavior to other languages, or we - # should always use Java server as backend. - if config.server_lang != skips.Lang.JAVA: - return False - if config.client_lang == skips.Lang.NODE: - return config.version_gte("v1.4.x") - return True - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - rpc_types=(RpcTypeUnaryCall,), - # UnaryCall only with sleep-2; timeout=1s; calls timeout. - metadata=((RpcTypeUnaryCall, "rpc-behavior", "sleep-2"),), - app_timeout=1, - num_rpcs=_NUM_RPCS, - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, - status_code=grpc.StatusCode.DEADLINE_EXCEEDED, - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_ERROR_TOLERANCE, - ) - - -class TestTimeoutNotExceeded(_BaseXdsTimeOutTestCase): - @staticmethod - def is_supported(config: skips.TestConfig) -> bool: - if config.client_lang == skips.Lang.NODE: - return config.version_gte("v1.4.x") - return True - - def rpc_distribution_validate(self, test_client: XdsTestClient): - self.configure_and_send( - test_client, - # UnaryCall only with no sleep; calls succeed. - rpc_types=(RpcTypeUnaryCall,), - num_rpcs=_NUM_RPCS, - ) - self.assertRpcStatusCode( - test_client, - expected=( - ExpectedResult( - rpc_type=RpcTypeUnaryCall, status_code=grpc.StatusCode.OK - ), - ), - length=_LENGTH_OF_RPC_SENDING_SEC, - tolerance=_ERROR_TOLERANCE, - ) - - -def load_tests(loader: absltest.TestLoader, unused_tests, unused_pattern): - suite = unittest.TestSuite() - test_cases = [ - TestTimeoutInRouteRule, - TestTimeoutInApplication, - TestTimeoutNotExceeded, - ] - for test_class in test_cases: - tests = loader.loadTestsFromTestCase(test_class) - suite.addTests(tests) - return suite - - -if __name__ == "__main__": - absltest.main()