Employ prebuilt images in continuous build. (#26184)

* Employ prebuilt images in continuous build. This commits updated Kokoro build job to use prebuilt images to run tests. The loadtest template was generated using loadtest_template.py.
4 years ago · 8a2754487c
parent 9990ce0dc4
commit 8a2754487c
2 changed files with 190 additions and 7 deletions
--- a/tools/internal_ci/linux/grpc_e2e_performance_v2.sh
+++ b/tools/internal_ci/linux/grpc_e2e_performance_v2.sh
@ -29,21 +29,59 @@ gcloud config set project grpc-testing
 gcloud container clusters get-credentials benchmarks-prod \
    --zone us-central1-b --project grpc-testing

+# Set up environment variables
+PREBUILT_IMAGE_PREFIX="gcr.io/grpc-testing/e2etesting/pre_built_workers/"${KOKORO_BUILD_INITIATOR}
+UNIQUE_IDENTIFIER=$(date +%Y%m%d%H%M%S)
+ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/"
+GRPC_CORE_GITREF="$(git ls-remote https://github.com/grpc/grpc.git master | cut -c1-7)"
+GRPC_GO_GITREF="$(git ls-remote https://github.com/grpc/grpc-go.git master | cut -c1-7)"
+GRPC_JAVA_GITREF="$(git ls-remote https://github.com/grpc/grpc-java.git master | cut -c1-7)"
+
+
+# Clone test-infra repository to one upper level directory than grpc
+cd ..
+git clone --recursive https://github.com/grpc/test-infra.git
+cd grpc
+
+# If there is a error within the function, will exit directly
+deleteImages() {
+  echo "an error has occurred after pushing the images, deleting images"
+  go run ../test-infra/tools/delete_prebuilt_workers/delete_prebuilt_workers.go \
+  -p $PREBUILT_IMAGE_PREFIX \
+  -t $UNIQUE_IDENTIFIER
+}
+trap deleteImages EXIT
+
+# Build and push prebuilt images for running tests
+go run ../test-infra/tools/prepare_prebuilt_workers/prepare_prebuilt_workers.go \
+  -l cxx:$GRPC_CORE_GITREF \
+  -l csharp:$GRPC_CORE_GITREF \
+  -l go:$GRPC_GO_GITREF \
+  -l java:$GRPC_JAVA_GITREF \
+  -l python:$GRPC_CORE_GITREF \
+  -l ruby:$GRPC_CORE_GITREF \
+  -p $PREBUILT_IMAGE_PREFIX \
+  -t $UNIQUE_IDENTIFIER \
+  -r $ROOT_DIRECTORY_OF_DOCKERFILES
+
 # This is subject to change. Runs a single test and does not wait for the
 # result.
 tools/run_tests/performance/loadtest_config.py -l c++ -l go \
    -t ./tools/run_tests/performance/templates/loadtest_template_basic_all_languages.yaml \
    -s client_pool=workers-8core -s server_pool=workers-8core \
    -s big_query_table=e2e_benchmarks.experimental_results \
-    -s timeout_seconds=900 --prefix="kokoro-test" -u "$(date +%Y%m%d%H%M%S)" \
+    -s timeout_seconds=900 \
+    -s prebuilt_image_prefix=$PREBUILT_IMAGE_PREFIX \
+    -s prebuilt_image_tag=$UNIQUE_IDENTIFIER \
+    --prefix=$KOKORO_BUILD_INITIATOR -u $UNIQUE_IDENTIFIER \
    -r '(go_generic_sync_streaming_ping_pong_secure|go_protobuf_sync_unary_ping_pong_secure|cpp_protobuf_async_streaming_qps_unconstrained_secure)$' \
-    -o ./loadtest.yaml
+    -o ./loadtest_with_prebuilt_images.yaml

-# Dump the contents of the loadtest.yaml (since loadtest_config.py doesn't
-# list the scenarios that will be run).
-cat ./loadtest.yaml
+# Dump the contents of the loadtest_with_prebuilt_images.yaml (since
+# loadtest_config.py doesn't list the scenarios that will be run).
+cat ./loadtest_with_prebuilt_images.yaml

-# The original version of the client is a bit old, update to the latest release
+# The original version of the client is old, update to the latest release
 # version v1.21.0.
 kubectl version --client
 curl -sSL -O https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl
@ -52,4 +90,4 @@ chmod +x kubectl
 sudo mv kubectl $(which kubectl)
 kubectl version --client

-kubectl apply -f ./loadtest.yaml
+kubectl apply -f ./loadtest_with_prebuilt_images.yaml
--- a/tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml
+++ b/tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml
@ -0,0 +1,145 @@
+# Template generated from load test configurations by loadtest_template.py.
+#
+# Configuration templates contain client and server configurations for multiple
+# languages, and may contain template substitution keys. These templates are
+# used to generate load test configurations by selecting clients and servers for
+# the required languages. The source files for template generation may be load
+# test configurations or load test configuration templates. Load test
+# configuration generation is performed by loadtest_config.py. See documentation
+# below:
+# https://github.com/grpc/grpc/blob/master/tools/run_tests/performance/README.md
+apiVersion: e2etest.grpc.io/v1
+kind: LoadTest
+metadata:
+  name: prebuilt_all_languages
+spec:
+  clients:
+    - language: csharp
+      pool: ${client_pool}
+      run:
+        args:
+          - exec
+          - /execute/qps_worker/Grpc.IntegrationTesting.QpsWorker.dll
+        command:
+          - dotnet
+        image: ${prebuilt_image_prefix}/csharp:${prebuilt_image_tag}
+    - language: cxx
+      pool: ${client_pool}
+      run:
+        command:
+          - /source/code/bazel-bin/test/cpp/qps/qps_worker
+        image: ${prebuilt_image_prefix}/cxx:${prebuilt_image_tag}
+    - language: go
+      pool: ${client_pool}
+      run:
+        command:
+          - /executable/bin/worker
+        image: ${prebuilt_image_prefix}/go:${prebuilt_image_tag}
+    - language: java
+      pool: ${client_pool}
+      run:
+        command:
+          - /execute/bin/benchmark_worker
+        image: ${prebuilt_image_prefix}/java:${prebuilt_image_tag}
+    - language: node
+      pool: ${client_pool}
+      run:
+        command:
+          - /execute/worker-linux
+          - --benchmark_impl=grpc
+        image: ${prebuilt_image_prefix}/node:${prebuilt_image_tag}
+    - language: php
+      pool: ${client_pool}
+      run:
+        command:
+          - bash
+          - /run_scripts/run_worker.sh
+        image: ${prebuilt_image_prefix}/php:${prebuilt_image_tag}
+    - language: python
+      pool: ${client_pool}
+      run:
+        command:
+          - /execute/qps_worker
+        image: ${prebuilt_image_prefix}/python:${prebuilt_image_tag}
+    - language: python_asyncio
+      pool: ${client_pool}
+      run:
+        command:
+          - /execute/benchmark_worker
+        image: ${prebuilt_image_prefix}/python:${prebuilt_image_tag}
+    - language: ruby
+      pool: ${client_pool}
+      run:
+        args:
+          - /execute/src/ruby/qps/worker.rb
+        command:
+          - ruby
+        image: ${prebuilt_image_prefix}/ruby:${prebuilt_image_tag}
+  results:
+    bigQueryTable: ${big_query_table}
+  servers:
+    - language: csharp
+      pool: ${server_pool}
+      run:
+        args:
+          - exec
+          - /execute/qps_worker/Grpc.IntegrationTesting.QpsWorker.dll
+        command:
+          - dotnet
+        image: ${prebuilt_image_prefix}/csharp:${prebuilt_image_tag}
+    - language: cxx
+      pool: ${server_pool}
+      run:
+        args:
+          - --server_port=10010
+        command:
+          - /source/code/bazel-bin/test/cpp/qps/qps_worker
+        image: ${prebuilt_image_prefix}/cxx:${prebuilt_image_tag}
+    - language: go
+      pool: ${server_pool}
+      run:
+        command:
+          - /executable/bin/worker
+        image: ${prebuilt_image_prefix}/go:${prebuilt_image_tag}
+    - language: java
+      pool: ${server_pool}
+      run:
+        command:
+          - /execute/bin/benchmark_worker
+        image: ${prebuilt_image_prefix}/java:${prebuilt_image_tag}
+    - language: node
+      pool: ${server_pool}
+      run:
+        command:
+          - /execute/worker-linux
+          - --benchmark_impl=grpc
+        image: ${prebuilt_image_prefix}/node:${prebuilt_image_tag}
+    - language: php
+      pool: ${server_pool}
+      run:
+        command:
+          - bash
+          - /run_scripts/run_worker.sh
+        image: ${prebuilt_image_prefix}/php:${prebuilt_image_tag}
+    - language: python
+      pool: ${server_pool}
+      run:
+        command:
+          - /execute/qps_worker
+        image: ${prebuilt_image_prefix}/python:${prebuilt_image_tag}
+    - language: python_asyncio
+      pool: ${server_pool}
+      run:
+        command:
+          - /execute/benchmark_worker
+        image: ${prebuilt_image_prefix}/python:${prebuilt_image_tag}
+    - language: ruby
+      pool: ${server_pool}
+      run:
+        args:
+          - /execute/src/ruby/qps/worker.rb
+        command:
+          - ruby
+        image: ${prebuilt_image_prefix}/ruby:${prebuilt_image_tag}
+  timeoutSeconds: ${timeout_seconds}
+  ttlSeconds: 86400