diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index a8da8a9fae6..6525bd5e54a 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -19,7 +19,6 @@ cd "$(dirname "$0")/../../.." source tools/internal_ci/helper_scripts/prepare_build_linux_rc - # Environment variables to select repos and branches for various repos. # You can edit these lines if you want to run from a fork. GRPC_CORE_REPO=grpc/grpc @@ -43,20 +42,20 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro && "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - # Use "official" BQ tables only for builds initiated by Kokoro and running - # from grpc/grpc. These results show up in the "official" public dashboard. - BIGQUERY_TABLE_8CORE=e2e_benchmarks.ci_master_results_8core - BIGQUERY_TABLE_32CORE=e2e_benchmarks.ci_master_results_32core + # Use "official" BQ tables only for builds initiated by Kokoro and running + # from grpc/grpc. These results show up in the "official" public dashboard. + BIGQUERY_TABLE_8CORE=e2e_benchmarks.ci_master_results_8core + BIGQUERY_TABLE_32CORE=e2e_benchmarks.ci_master_results_32core else - # Use experimental BQ tables otherwise. - BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results - BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core + # Use experimental BQ tables otherwise. + BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results + BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core fi # END differentiate experimental configuration from master configuration. CLOUD_LOGGING_URL="https://source.cloud.google.com/results/invocations/${KOKORO_BUILD_ID}" @@ -67,9 +66,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi GRPC_DOTNET_COMMIT="$(git ls-remote "https://github.com/${GRPC_DOTNET_REPO}.git" "${GRPC_DOTNET_GITREF}" | cut -f1)" @@ -94,76 +93,109 @@ popd # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ - -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ - -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ - --allow_client_language=c++ --allow_server_language=c++ \ - --allow_server_language=node \ - -o "loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ + -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ + -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=scalable \ + --allow_client_language=c++ --allow_server_language=c++ \ + --allow_server_language=node \ + -o "loadtest_with_prebuilt_workers_${pool}.yaml" } -# Add languages +# List all languages. +declare -A useLanguage=( + [c++]=1 + [dotnet]=1 + [go]=1 + [java]=1 + [node]=1 + [ruby]=1 +) + +# Disable specific languages. +declare -a disabledLanguages=( + # Add a language here to disable it. + dotnet +) +for language in "${disabledLanguages[@]}"; do + unset "useLanguage[${language}]" +done + +# Add arguments for languages. declare -a configLangArgs8core=() declare -a configLangArgs32core=() declare -a runnerLangArgs=() # c++ -configLangArgs8core+=( -l c++ ) -configLangArgs32core+=( -l c++ ) -runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[c++]" ]]; then + configLangArgs8core+=(-l c++) + configLangArgs32core+=(-l c++) + runnerLangArgs+=(-l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi # dotnet -# configLangArgs8core+=( -l dotnet ) -# configLangArgs32core+=( -l dotnet ) -# runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) +if [[ -v "useLanguage[dotnet]" ]]; then + configLangArgs8core+=(-l dotnet) + configLangArgs32core+=(-l dotnet) + runnerLangArgs+=(-l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}") +fi -# # go -configLangArgs8core+=( -l go ) -configLangArgs32core+=( -l go ) -runnerLangArgs+=( -l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}" ) +# go +if [[ -v "useLanguage[go]" ]]; then + configLangArgs8core+=(-l go) + configLangArgs32core+=(-l go) + runnerLangArgs+=(-l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}") +fi # java -configLangArgs8core+=( -l java ) -configLangArgs32core+=( -l java ) -runnerLangArgs+=( -l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}" ) +if [[ -v "useLanguage[java]" ]]; then + configLangArgs8core+=(-l java) + configLangArgs32core+=(-l java) + runnerLangArgs+=(-l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}") +fi # node -configLangArgs8core+=( -l node ) # 8-core only. -runnerLangArgs+=( -l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}" ) +if [[ -v "useLanguage[node]" ]]; then + configLangArgs8core+=(-l node) # 8-core only. + runnerLangArgs+=(-l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}") +fi # python -configLangArgs8core+=( -l python ) # 8-core only. -runnerLangArgs+=( -l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[python]" ]]; then + configLangArgs8core+=(-l python) # 8-core only. + runnerLangArgs+=(-l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi # ruby -configLangArgs8core+=( -l ruby ) # 8-core only. -runnerLangArgs+=( -l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[ruby]" ]]; then + configLangArgs8core+=(-l ruby) # 8-core only. + runnerLangArgs+=(-l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${configLangArgs8core[@]}" buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${configLangArgs32core[@]}" # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -171,16 +203,16 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers "${runnerLangArgs[@]}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh index dfad208da8c..723369b2580 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_cxx_experiments_framework.sh @@ -18,9 +18,9 @@ set -ex # # To run the benchmarks, add your experiment to the set below. declare -a GRPC_EXPERIMENTS=( - "event_engine_listener" - "event_engine_client" - "event_engine_client,event_engine_listener" + "event_engine_listener" + "event_engine_client" + "event_engine_client,event_engine_listener" ) # Enter the gRPC repo root. @@ -43,13 +43,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro + LOAD_TEST_PREFIX=kokoro fi # Use the "official" BQ tables so that the measurements will show up in the # "official" public dashboard. @@ -64,9 +64,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi # Kokoro jobs run on dedicated pools. @@ -89,62 +89,62 @@ declare -a loadtest_files=() # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r base_table="$2" - local -r experiment="$3" - shift 3 - # Multiple experiments are delimited by `__` (two underscores) in BigQuery. - SANITIZED_EXPERIMENT="${experiment//,/__}" - OUTFILE="loadtest_with_prebuilt_workers_${pool}_${SANITIZED_EXPERIMENT}.yaml" - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_cxx_experiments.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${base_table}_${SANITIZED_EXPERIMENT}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -s grpc_experiment="${experiment}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=dashboard \ - --allow_client_language=c++ --allow_server_language=c++ \ - --allow_server_language=node \ - -o "${OUTFILE}" - - loadtest_files+=(-i "${OUTFILE}") + local -r pool="$1" + local -r base_table="$2" + local -r experiment="$3" + shift 3 + # Multiple experiments are delimited by `__` (two underscores) in BigQuery. + SANITIZED_EXPERIMENT="${experiment//,/__}" + OUTFILE="loadtest_with_prebuilt_workers_${pool}_${SANITIZED_EXPERIMENT}.yaml" + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_cxx_experiments.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${base_table}_${SANITIZED_EXPERIMENT}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -s grpc_experiment="${experiment}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=dashboard \ + --allow_client_language=c++ --allow_server_language=c++ \ + --allow_server_language=node \ + -o "${OUTFILE}" + + loadtest_files+=(-i "${OUTFILE}") } for experiment in "${GRPC_EXPERIMENTS[@]}"; do - buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${experiment}" -l c++ - buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${experiment}" -l c++ + buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${experiment}" -l c++ + buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${experiment}" -l c++ done # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" } trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ - -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. ../test-infra/bin/runner \ - ${loadtest_files[@]} \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + "${loadtest_files[@]}" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh index c3b99af80eb..f15a716028a 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh @@ -19,7 +19,6 @@ cd "$(dirname "$0")/../../.." source tools/internal_ci/helper_scripts/prepare_build_linux_rc - # Environment variables to select repos and branches for various repos. # You can edit these lines if you want to run from a fork. GRPC_CORE_REPO=grpc/grpc @@ -43,13 +42,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials benchmarks-prod2 \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro-test + LOAD_TEST_PREFIX=kokoro-test fi BIGQUERY_TABLE_8CORE=e2e_benchmarks.experimental_results BIGQUERY_TABLE_32CORE=e2e_benchmarks.experimental_results_32core @@ -62,9 +61,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_COMMIT="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" + GRPC_CORE_COMMIT="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" + GRPC_CORE_COMMIT="$(git ls-remote -h "https://github.com/${GRPC_CORE_REPO}.git" "${GRPC_CORE_GITREF}" | cut -f1)" fi GRPC_DOTNET_COMMIT="$(git ls-remote "https://github.com/${GRPC_DOTNET_REPO}.git" "${GRPC_DOTNET_GITREF}" | cut -f1)" @@ -89,75 +88,109 @@ popd # Build test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_COMMIT}" \ - -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ - -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ - -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ - -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_COMMIT}" \ + -a ci_gitCommit_core="${GRPC_CORE_COMMIT}" \ + -a ci_gitCommit_dotnet="${GRPC_DOTNET_COMMIT}" \ + -a ci_gitCommit_go="${GRPC_GO_COMMIT}" \ + -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=scalable \ + --allow_client_language=c++ --allow_server_language=c++ \ + --allow_server_language=node \ + -o "loadtest_with_prebuilt_workers_${pool}.yaml" } -# Add languages +# List all languages. +declare -A useLanguage=( + [c++]=1 + [dotnet]=1 + [go]=1 + [java]=1 + [node]=1 + [ruby]=1 +) + +# Disable specific languages. +declare -a disabledLanguages=( + # Add a language here to disable it. + dotnet +) +for language in "${disabledLanguages[@]}"; do + unset "useLanguage[${language}]" +done + +# Add arguments for languages. declare -a configLangArgs8core=() declare -a configLangArgs32core=() declare -a runnerLangArgs=() # c++ -configLangArgs8core+=( -l c++ ) -configLangArgs32core+=( -l c++ ) -runnerLangArgs+=( -l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[c++]" ]]; then + configLangArgs8core+=(-l c++) + configLangArgs32core+=(-l c++) + runnerLangArgs+=(-l "cxx:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi # dotnet -# configLangArgs8core+=( -l dotnet ) -# configLangArgs32core+=( -l dotnet ) -# runnerLangArgs+=( -l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}" ) +if [[ -v "useLanguage[dotnet]" ]]; then + configLangArgs8core+=(-l dotnet) + configLangArgs32core+=(-l dotnet) + runnerLangArgs+=(-l "dotnet:${GRPC_DOTNET_REPO}:${GRPC_DOTNET_COMMIT}") +fi -# # go -configLangArgs8core+=( -l go ) -configLangArgs32core+=( -l go ) -runnerLangArgs+=( -l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}" ) +# go +if [[ -v "useLanguage[go]" ]]; then + configLangArgs8core+=(-l go) + configLangArgs32core+=(-l go) + runnerLangArgs+=(-l "go:${GRPC_GO_REPO}:${GRPC_GO_COMMIT}") +fi # java -configLangArgs8core+=( -l java ) -configLangArgs32core+=( -l java ) -runnerLangArgs+=( -l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}" ) +if [[ -v "useLanguage[java]" ]]; then + configLangArgs8core+=(-l java) + configLangArgs32core+=(-l java) + runnerLangArgs+=(-l "java:${GRPC_JAVA_REPO}:${GRPC_JAVA_COMMIT}") +fi # node -configLangArgs8core+=( -l node ) # 8-core only. -runnerLangArgs+=( -l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}" ) +if [[ -v "useLanguage[node]" ]]; then + configLangArgs8core+=(-l node) # 8-core only. + runnerLangArgs+=(-l "node:${GRPC_NODE_REPO}:${GRPC_NODE_COMMIT}") +fi # python -configLangArgs8core+=( -l python ) # 8-core only. -runnerLangArgs+=( -l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[python]" ]]; then + configLangArgs8core+=(-l python) # 8-core only. + runnerLangArgs+=(-l "python:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi # ruby -configLangArgs8core+=( -l ruby ) # 8-core only. -runnerLangArgs+=( -l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}" ) +if [[ -v "useLanguage[ruby]" ]]; then + configLangArgs8core+=(-l ruby) # 8-core only. + runnerLangArgs+=(-l "ruby:${GRPC_CORE_REPO}:${GRPC_CORE_COMMIT}") +fi buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" "${configLangArgs8core[@]}" buildConfigs "${WORKER_POOL_32CORE}" "${BIGQUERY_TABLE_32CORE}" "${configLangArgs32core[@]}" # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -165,16 +198,16 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers "${runnerLangArgs[@]}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ - -o "runner/sponge_log.xml" + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "loadtest_with_prebuilt_workers_${WORKER_POOL_32CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}:2" -c "${WORKER_POOL_32CORE}:2" \ + -o "runner/sponge_log.xml" diff --git a/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh index 143e923359b..44a67643838 100644 --- a/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_psm_performance_gke_experiment.sh @@ -27,13 +27,13 @@ gcloud auth configure-docker # Connect to benchmarks-prod2 cluster. gcloud config set project grpc-testing gcloud container clusters get-credentials psm-benchmarks-performance \ - --zone us-central1-b --project grpc-testing + --zone us-central1-b --project grpc-testing # Set up environment variables. LOAD_TEST_PREFIX="${KOKORO_BUILD_INITIATOR}" # BEGIN differentiate experimental configuration from master configuration. if [[ "${KOKORO_BUILD_INITIATOR%%-*}" == kokoro ]]; then - LOAD_TEST_PREFIX=kokoro-test + LOAD_TEST_PREFIX=kokoro-test fi BIGQUERY_TABLE_8CORE=e2e_benchmarks.psm_experimental_results_8core # END differentiate experimental configuration from master configuration. @@ -45,9 +45,9 @@ ROOT_DIRECTORY_OF_DOCKERFILES="../test-infra/containers/pre_built_workers/" GRPC_GITREF="$(git show --format="%H" --no-patch)" # Prebuilt workers for core languages are always built from grpc/grpc. if [[ "${KOKORO_GITHUB_COMMIT_URL%/*}" == "https://github.com/grpc/grpc/commit" ]]; then - GRPC_CORE_GITREF="${KOKORO_GIT_COMMIT}" + GRPC_CORE_GITREF="${KOKORO_GIT_COMMIT}" else - GRPC_CORE_GITREF="$(git ls-remote -h https://github.com/grpc/grpc.git master | cut -f1)" + GRPC_CORE_GITREF="$(git ls-remote -h https://github.com/grpc/grpc.git master | cut -f1)" fi GRPC_JAVA_GITREF="$(git ls-remote -h https://github.com/grpc/grpc-java.git master | cut -f1)" # Kokoro jobs run on dedicated pools. @@ -75,69 +75,69 @@ PSM_IMAGE_TAG=${TEST_INFRA_VERSION} # Build psm test configurations. psmBuildConfigs() { - local -r pool="$1" - local -r table="$2" - local -r proxy_type="$3" - - shift 3 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_psm_"${proxy_type}"_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -s psm_image_prefix="${PSM_IMAGE_PREFIX}" \ - -s psm_image_tag="${PSM_IMAGE_TAG}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_GITREF}" \ - -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - -a enablePrometheus=true \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" -u "${proxy_type}"\ - -a pool="${pool}" \ - --category=psm \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "psm_${proxy_type}_loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + local -r proxy_type="$3" + + shift 3 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_psm_"${proxy_type}"_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -s psm_image_prefix="${PSM_IMAGE_PREFIX}" \ + -s psm_image_tag="${PSM_IMAGE_TAG}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_GITREF}" \ + -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + -a enablePrometheus=true \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" -u "${proxy_type}" \ + -a pool="${pool}" \ + --category=psm \ + --allow_client_language=c++ --allow_server_language=c++ \ + -o "psm_${proxy_type}_loadtest_with_prebuilt_workers_${pool}.yaml" } -psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxied -a queue="${WORKER_POOL_8CORE}-proxied" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 +psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxied -a queue="${WORKER_POOL_8CORE}-proxied" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 psmBuildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" proxyless -a queue="${WORKER_POOL_8CORE}-proxyless" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 # Build regular test configurations. buildConfigs() { - local -r pool="$1" - local -r table="$2" - shift 2 - tools/run_tests/performance/loadtest_config.py "$@" \ - -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ - -s driver_pool="${DRIVER_POOL}" -s driver_image= \ - -s client_pool="${pool}" -s server_pool="${pool}" \ - -s big_query_table="${table}" -s timeout_seconds=900 \ - -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ - -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ - -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ - -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ - -a ci_jobName="${KOKORO_JOB_NAME}" \ - -a ci_gitCommit="${GRPC_GITREF}" \ - -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ - -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ - -a enablePrometheus=true \ - --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=psm \ - --allow_client_language=c++ --allow_server_language=c++ \ - -o "regular_loadtest_with_prebuilt_workers_${pool}.yaml" + local -r pool="$1" + local -r table="$2" + shift 2 + tools/run_tests/performance/loadtest_config.py "$@" \ + -t ./tools/run_tests/performance/templates/loadtest_template_prebuilt_all_languages.yaml \ + -s driver_pool="${DRIVER_POOL}" -s driver_image= \ + -s client_pool="${pool}" -s server_pool="${pool}" \ + -s big_query_table="${table}" -s timeout_seconds=900 \ + -s prebuilt_image_prefix="${PREBUILT_IMAGE_PREFIX}" \ + -s prebuilt_image_tag="${UNIQUE_IDENTIFIER}" \ + -a ci_buildNumber="${KOKORO_BUILD_NUMBER}" \ + -a ci_buildUrl="${CLOUD_LOGGING_URL}" \ + -a ci_jobName="${KOKORO_JOB_NAME}" \ + -a ci_gitCommit="${GRPC_GITREF}" \ + -a ci_gitCommit_java="${GRPC_JAVA_GITREF}" \ + -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ + -a enablePrometheus=true \ + --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ + -a pool="${pool}" --category=psm \ + --allow_client_language=c++ --allow_server_language=c++ \ + -o "regular_loadtest_with_prebuilt_workers_${pool}.yaml" } buildConfigs "${WORKER_POOL_8CORE}" "${BIGQUERY_TABLE_8CORE}" -a queue="${WORKER_POOL_8CORE}-regular" -l c++ -l java --client_channels=8 --server_threads=16 --offered_loads 100 300 500 700 900 1000 1500 2000 2500 4000 6000 8000 10000 12000 14000 16000 18000 20000 22000 24000 26000 28000 30000 # Delete prebuilt images on exit. deleteImages() { - echo "deleting images on exit" - ../test-infra/bin/delete_prebuilt_workers \ + echo "deleting images on exit" + ../test-infra/bin/delete_prebuilt_workers \ -p "${PREBUILT_IMAGE_PREFIX}" \ -t "${UNIQUE_IDENTIFIER}" } @@ -145,22 +145,22 @@ trap deleteImages EXIT # Build and push prebuilt images for running tests. time ../test-infra/bin/prepare_prebuilt_workers \ - -l "cxx:${GRPC_CORE_GITREF}" \ - -l "java:${GRPC_JAVA_GITREF}" \ - -p "${PREBUILT_IMAGE_PREFIX}" \ - -t "${UNIQUE_IDENTIFIER}" \ - -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" + -l "cxx:${GRPC_CORE_GITREF}" \ + -l "java:${GRPC_JAVA_GITREF}" \ + -p "${PREBUILT_IMAGE_PREFIX}" \ + -t "${UNIQUE_IDENTIFIER}" \ + -r "${ROOT_DIRECTORY_OF_DOCKERFILES}" # Run tests. time ../test-infra/bin/runner \ - -i "psm_proxied_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "psm_proxyless_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -i "regular_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ - -log-url-prefix "${LOG_URL_PREFIX}" \ - -annotation-key queue \ - -polling-interval 5s \ - -delete-successful-tests \ - -c "${WORKER_POOL_8CORE}-proxied:1" \ - -c "${WORKER_POOL_8CORE}-proxyless:1" \ - -c "${WORKER_POOL_8CORE}-regular:1" \ - -o "runner/sponge_log.xml" + -i "psm_proxied_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "psm_proxyless_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -i "regular_loadtest_with_prebuilt_workers_${WORKER_POOL_8CORE}.yaml" \ + -log-url-prefix "${LOG_URL_PREFIX}" \ + -annotation-key queue \ + -polling-interval 5s \ + -delete-successful-tests \ + -c "${WORKER_POOL_8CORE}-proxied:1" \ + -c "${WORKER_POOL_8CORE}-proxyless:1" \ + -c "${WORKER_POOL_8CORE}-regular:1" \ + -o "runner/sponge_log.xml"