Integrate test runner into kokoro performance runs. (#26225)

tools/internal_ci/linux/grpc_e2e_performance_v2.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 set -ex
 
-# Enter the gRPC repo root
+# Enter the gRPC repo root.
 cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
@@ -29,65 +29,65 @@ gcloud config set project grpc-testing
 gcloud container clusters get-credentials benchmarks-prod \
     --zone us-central1-b --project grpc-testing
 
-# Set up environment variables
-PREBUILT_IMAGE_PREFIX="gcr.io/grpc-testing/e2etesting/pre_built_workers/"${KOKORO_BUILD_INITIATOR}
+# Set up environment variables.
+PREBUILT_IMAGE_PREFIX="gcr.io/grpc-testing/e2etesting/pre_built_workers/${KOKORO_BUILD_INITIATOR}"
 UNIQUE_IDENTIFIER=$(date +%Y%m%d%H%M%S)
 ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/"
 GRPC_CORE_GITREF="$(git ls-remote https://github.com/grpc/grpc.git master | cut -c1-7)"
 GRPC_GO_GITREF="$(git ls-remote https://github.com/grpc/grpc-go.git master | cut -c1-7)"
 GRPC_JAVA_GITREF="$(git ls-remote https://github.com/grpc/grpc-java.git master | cut -c1-7)"
 
-
-# Clone test-infra repository to one upper level directory than grpc
-cd ..
+# Clone test-infra repository to one upper level directory than grpc.
+pushd ..
 git clone --recursive https://github.com/grpc/test-infra.git
-cd grpc
+cd test-infra
+go build -o bin/runner cmd/runner/main.go
+popd
 
-# If there is a error within the function, will exit directly
-deleteImages() {
-  echo "an error has occurred after pushing the images, deleting images"
-  go run ../test-infra/tools/delete_prebuilt_workers/delete_prebuilt_workers.go \
-  -p $PREBUILT_IMAGE_PREFIX \
-  -t $UNIQUE_IDENTIFIER
+# Build test configurations.
+buildConfigs() {
+    local pool="$1"
+    shift
+    tools/run_tests/performance/loadtest_config.py "$@" \
+        -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \
+        -s client_pool="${pool}" -s server_pool="${pool}" \
+        -s big_query_table=e2e_benchmarks.experimental_results \
+        -s timeout_seconds=900 \
+        -s prebuilt_image_prefix=$PREBUILT_IMAGE_PREFIX \
+        -s prebuilt_image_tag=$UNIQUE_IDENTIFIER \
+        --prefix=$KOKORO_BUILD_INITIATOR -u $UNIQUE_IDENTIFIER -u "${pool}" \
+        -a pool="${pool}" --category=scalable \
+        --allow_client_language=c++ --allow_server_language=c++ \
+        -o "./loadtest_with_prebuilt_workers_${pool}.yaml"
 }
-trap deleteImages EXIT
 
-# Build and push prebuilt images for running tests
-go run ../test-infra/tools/prepare_prebuilt_workers/prepare_prebuilt_workers.go \
-  -l cxx:$GRPC_CORE_GITREF \
-  -l csharp:$GRPC_CORE_GITREF \
-  -l go:$GRPC_GO_GITREF \
-  -l java:$GRPC_JAVA_GITREF \
-  -l python:$GRPC_CORE_GITREF \
-  -l ruby:$GRPC_CORE_GITREF \
-  -p $PREBUILT_IMAGE_PREFIX \
-  -t $UNIQUE_IDENTIFIER \
-  -r $ROOT_DIRECTORY_OF_DOCKERFILES
+buildConfigs workers-8core -l c++ -l csharp -l go -l java -l python -l ruby
 
-# This is subject to change. Runs a single test and does not wait for the
-# result.
-tools/run_tests/performance/loadtest_config.py -l c++ -l go \
-    -t ./tools/run_tests/performance/templates/loadtest_template_basic_all_languages.yaml \
-    -s client_pool=workers-8core -s server_pool=workers-8core \
-    -s big_query_table=e2e_benchmarks.experimental_results \
-    -s timeout_seconds=900 \
-    -s prebuilt_image_prefix=$PREBUILT_IMAGE_PREFIX \
-    -s prebuilt_image_tag=$UNIQUE_IDENTIFIER \
-    --prefix=$KOKORO_BUILD_INITIATOR -u $UNIQUE_IDENTIFIER \
-    -r '(go_generic_sync_streaming_ping_pong_secure|go_protobuf_sync_unary_ping_pong_secure|cpp_protobuf_async_streaming_qps_unconstrained_secure)$' \
-    -o ./loadtest_with_prebuilt_images.yaml
+buildConfigs workers-32core -l c++ -l csharp -l go -l java
 
-# Dump the contents of the loadtest_with_prebuilt_images.yaml (since
-# loadtest_config.py doesn't list the scenarios that will be run).
-cat ./loadtest_with_prebuilt_images.yaml
+# Delete prebuilt images on exit.
+deleteImages() {
+    echo "deleting images on exit"
+    go run ../test-infra/tools/delete_prebuilt_workers/delete_prebuilt_workers.go \
+    -p $PREBUILT_IMAGE_PREFIX \
+    -t $UNIQUE_IDENTIFIER
+}
+trap deleteImages EXIT
 
-# The original version of the client is old, update to the latest release
-# version v1.21.0.
-kubectl version --client
-curl -sSL -O https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl
-sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-chmod +x kubectl
-sudo mv kubectl $(which kubectl)
-kubectl version --client
+# Build and push prebuilt images for running tests.
+go run ../test-infra/tools/prepare_prebuilt_workers/prepare_prebuilt_workers.go \
+    -l cxx:$GRPC_CORE_GITREF \
+    -l csharp:$GRPC_CORE_GITREF \
+    -l go:$GRPC_GO_GITREF \
+    -l java:$GRPC_JAVA_GITREF \
+    -l python:$GRPC_CORE_GITREF \
+    -l ruby:$GRPC_CORE_GITREF \
+    -p $PREBUILT_IMAGE_PREFIX \
+    -t $UNIQUE_IDENTIFIER \
+    -r $ROOT_DIRECTORY_OF_DOCKERFILES
 
-kubectl apply -f ./loadtest_with_prebuilt_images.yaml
+# Run tests.
+../test-infra/bin/runner \
+    -i ../grpc/loadtest_with_prebuilt_workers_workers-8core.yaml \
+    -i ../grpc/loadtest_with_prebuilt_workers_workers-32core.yaml \
+    -a pool -c workers-8core:8 -c workers-32core:8