From b0d4add3a0e95f12ca4d4086fdc790b5c84d5ade Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 1 Apr 2016 16:36:23 -0700 Subject: [PATCH 1/7] new config files --- tools/run_tests/stress_test/configs/asan.json | 86 +++++++++++++++++++ tools/run_tests/stress_test/configs/opt.json | 4 +- tools/run_tests/stress_test/configs/tsan.json | 86 +++++++++++++++++++ 3 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 tools/run_tests/stress_test/configs/asan.json create mode 100644 tools/run_tests/stress_test/configs/tsan.json diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json new file mode 100644 index 00000000000..4bbf0d420ed --- /dev/null +++ b/tools/run_tests/stress_test/configs/asan.json @@ -0,0 +1,86 @@ +{ + "dockerImages": { + "grpc_stress_cxx_asan" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_cxx", + "buildType": "asan" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081, + "metrics_collection_interval_secs":60 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + } + } + }, + "templates": { + "cxx_client_asan": { + "baseTemplate": "default", + "clientImagePath": "/var/local/git/grpc/bins/asan/stress_test", + "metricsClientImagePath": "/var/local/git/grpc/bins/asan/metrics_client" + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + } + } + }, + "templates": { + "cxx_server_asan": { + "baseTemplate": "default", + "serverImagePath": "/var/local/git/grpc/bins/asan/interop_server" + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "stress-server-asan": { + "serverTemplate": "cxx_server_asan", + "dockerImage": "grpc_stress_cxx_asan", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "stress-client-asan": { + "clientTemplate": "cxx_client_asan", + "dockerImage": "grpc_stress_cxx_asan", + "numInstances": 10, + "serverPodSpec": "stress-server-asan" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8001, + "datasetIdNamePrefix": "stress_test_asan", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index 7fc024034b8..b86c0a44210 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -74,8 +74,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 10, - "testDurationSecs": 120, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_opt", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json new file mode 100644 index 00000000000..182d167cadc --- /dev/null +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -0,0 +1,86 @@ +{ + "dockerImages": { + "grpc_stress_cxx_tsan" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_cxx", + "buildType": "tsan" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081, + "metrics_collection_interval_secs":60 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + } + } + }, + "templates": { + "cxx_client_tsan": { + "baseTemplate": "default", + "clientImagePath": "/var/local/git/grpc/bins/tsan/stress_test", + "metricsClientImagePath": "/var/local/git/grpc/bins/tsan/metrics_client" + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + } + } + }, + "templates": { + "cxx_server_tsan": { + "baseTemplate": "default", + "serverImagePath": "/var/local/git/grpc/bins/tsan/interop_server" + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "stress-server-tsan": { + "serverTemplate": "cxx_server_tsan", + "dockerImage": "grpc_stress_cxx_tsan", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "stress-client-tsan": { + "clientTemplate": "cxx_client_tsan", + "dockerImage": "grpc_stress_cxx_tsan", + "numInstances": 10, + "serverPodSpec": "stress-server-tsan" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8001, + "datasetIdNamePrefix": "stress_test_tsan", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + From 7a05436ac1a871719fb636c35b26e6f8592a2620 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 4 Apr 2016 14:08:02 -0700 Subject: [PATCH 2/7] New config for node. Made some changes to support node config --- .../Dockerfile.template | 40 ++++++++ .../Dockerfile.template | 9 +- .../grpc_interop_stress_cxx/Dockerfile | 0 .../build_interop_stress.sh | 0 .../grpc_interop_stress_node/Dockerfile | 95 +++++++++++++++++++ .../build_interop_stress.sh | 48 ++++++++++ tools/gcp/stress_test/run_client.py | 9 +- tools/gcp/stress_test/run_server.py | 4 +- tools/jenkins/build_interop_stress_image.sh | 8 +- tools/run_tests/stress_test/configs/asan.json | 12 +-- tools/run_tests/stress_test/configs/node.json | 84 ++++++++++++++++ .../stress_test/configs/opt-tsan-asan.json | 24 ++--- tools/run_tests/stress_test/configs/opt.json | 12 +-- tools/run_tests/stress_test/configs/tsan.json | 12 +-- tools/run_tests/stress_test/run_on_gke.py | 26 ++--- 15 files changed, 324 insertions(+), 59 deletions(-) create mode 100644 templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template rename templates/tools/dockerfile/{grpc_interop_stress_cxx => stress_test/grpc_interop_stress_node}/Dockerfile.template (88%) rename tools/dockerfile/{ => stress_test}/grpc_interop_stress_cxx/Dockerfile (100%) rename tools/dockerfile/{ => stress_test}/grpc_interop_stress_cxx/build_interop_stress.sh (100%) create mode 100644 tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile create mode 100755 tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh create mode 100644 tools/run_tests/stress_test/configs/node.json diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template new file mode 100644 index 00000000000..61b16c77159 --- /dev/null +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template @@ -0,0 +1,40 @@ +%YAML 1.2 +--- | + # Copyright 2015, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../ccache_setup.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../clang_update.include"/> + # Define the default command. + CMD ["bash"] diff --git a/templates/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template similarity index 88% rename from templates/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile.template rename to templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template index 5dd0da1a624..6520ec29c1d 100644 --- a/templates/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template @@ -31,10 +31,9 @@ FROM debian:jessie - <%include file="../apt_get_basic.include"/> - <%include file="../ccache_setup.include"/> - <%include file="../cxx_deps.include"/> - <%include file="../gcp_api_libraries.include"/> - <%include file="../clang_update.include"/> + <%include file="../../apt_get_basic.include"/> + <%include file="../../node_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile similarity index 100% rename from tools/dockerfile/grpc_interop_stress_cxx/Dockerfile rename to tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile diff --git a/tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh similarity index 100% rename from tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh rename to tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile new file mode 100644 index 00000000000..a932b4087d8 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile @@ -0,0 +1,95 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +#================== +# Node dependencies + +# Install nvm +RUN touch .profile +RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash +RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#====================== +# Zookeeper dependencies +# TODO(jtattermusch): is zookeeper still needed? +RUN apt-get install -y libzookeeper-mt-dev + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh new file mode 100755 index 00000000000..4d4290d0b41 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Node interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc +nvm use 0.12 +nvm alias default 0.12 # prevent the need to run 'nvm use' in every shell + +make install-certs + +# build Node interop client & server +npm install -g node-gyp +npm install --unsafe-perm --build-from-source diff --git a/tools/gcp/stress_test/run_client.py b/tools/gcp/stress_test/run_client.py index e6b28d56244..71ee0c7c3cc 100755 --- a/tools/gcp/stress_test/run_client.py +++ b/tools/gcp/stress_test/run_client.py @@ -91,9 +91,9 @@ def run_client(): """ env = dict(os.environ) image_type = env['STRESS_TEST_IMAGE_TYPE'] - image_name = env['STRESS_TEST_IMAGE'] + stress_client_cmd = env['STRESS_TEST_CMD'].split() args_str = env['STRESS_TEST_ARGS_STR'] - metrics_client_image = env['METRICS_CLIENT_IMAGE'] + metrics_client_cmd = env['METRICS_CLIENT_CMD'].split() metrics_client_args_str = env['METRICS_CLIENT_ARGS_STR'] run_id = env['RUN_ID'] pod_name = env['POD_NAME'] @@ -125,9 +125,8 @@ def run_client(): # Update status that the test is starting (in the status table) bq_helper.insert_summary_row(EventType.STARTING, details) - metrics_cmd = [metrics_client_image - ] + [x for x in metrics_client_args_str.split()] - stress_cmd = [image_name] + [x for x in args_str.split()] + metrics_cmd = metrics_client_cmd + [x for x in metrics_client_args_str.split()] + stress_cmd = stress_client_cmd + [x for x in args_str.split()] print 'Launching process %s ...' % stress_cmd stress_p = subprocess.Popen(args=stress_cmd, diff --git a/tools/gcp/stress_test/run_server.py b/tools/gcp/stress_test/run_server.py index dc4741b95eb..a02edf8ea41 100755 --- a/tools/gcp/stress_test/run_server.py +++ b/tools/gcp/stress_test/run_server.py @@ -62,7 +62,7 @@ def run_server(): run_id = env['RUN_ID'] # The unique run id for this test image_type = env['STRESS_TEST_IMAGE_TYPE'] - image_name = env['STRESS_TEST_IMAGE'] + stress_server_cmd = env['STRESS_TEST_CMD'].split() args_str = env['STRESS_TEST_ARGS_STR'] pod_name = env['POD_NAME'] project_id = env['GCP_PROJECT_ID'] @@ -98,7 +98,7 @@ def run_server(): # Update status that the test is starting (in the status table) bq_helper.insert_summary_row(EventType.STARTING, details) - stress_cmd = [image_name] + [x for x in args_str.split()] + stress_cmd = stress_server_cmd + [x for x in args_str.split()] print 'Launching process %s ...' % stress_cmd stress_p = subprocess.Popen(args=stress_cmd, diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh index 14621bf6c4f..29c8ed64271 100755 --- a/tools/jenkins/build_interop_stress_image.sh +++ b/tools/jenkins/build_interop_stress_image.sh @@ -35,7 +35,7 @@ set -x # Params: # INTEROP_IMAGE - Name of tag of the final interop image -# INTEROP_IMAGE_TAG - Optional. If set, the created image will be tagged using +# INTEROP_IMAGE_REPOSITORY_TAG - Optional. If set, the created image will be tagged using # the command: 'docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG' # BASE_NAME - Base name used to locate the base Dockerfile and build script # BUILD_TYPE - The 'CONFIG' variable passed to the 'make' command (example: @@ -59,10 +59,10 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/stress_test/$BASE_NAME/Dockerfile | cut -f1 -d\ ` # Make sure base docker image has been built. Should be instantaneous if so. -docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/$BASE_NAME || exit $? +docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/stress_test/$BASE_NAME || exit $? # Create a local branch so the child Docker script won't complain git branch -f jenkins-docker @@ -80,7 +80,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)" -v /tmp/ccache:/tmp/ccache \ --name=$CONTAINER_NAME \ $BASE_IMAGE \ - bash -l /var/local/jenkins/grpc/tools/dockerfile/$BASE_NAME/build_interop_stress.sh \ + bash -l /var/local/jenkins/grpc/tools/dockerfile/stress_test/$BASE_NAME/build_interop_stress.sh \ && docker commit $CONTAINER_NAME $INTEROP_IMAGE \ && ( if [ -n "$INTEROP_IMAGE_REPOSITORY_TAG" ]; then docker tag -f $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG ; fi ) \ && echo "Successfully built image $INTEROP_IMAGE") diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index 4bbf0d420ed..7414829da84 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":10 }, "metricsPort": 8081, "metricsArgs": { @@ -29,8 +29,8 @@ "templates": { "cxx_client_asan": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/asan/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/asan/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/asan/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/asan/metrics_client" } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_asan": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/asan/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/asan/interop_server" } } }, @@ -74,8 +74,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, + "pollIntervalSecs": 10, + "testDurationSecs": 60, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_asan", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json new file mode 100644 index 00000000000..525f9031b32 --- /dev/null +++ b/tools/run_tests/stress_test/configs/node.json @@ -0,0 +1,84 @@ +{ + "dockerImages": { + "grpc_stress_node" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_node" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + } + } + }, + "templates": { + "node_client": { + "baseTemplate": "default", + "stressClientCmd": "node /var/local/git/grpc/src/node/stress/stress_client.js", + "metricsClientCmd": "node /var/local/git/grpc/src/node/stress/metrics_client.js" + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + } + } + }, + "templates": { + "node_server": { + "baseTemplate": "default", + "stressServerCmd": "node /var/local/git/grpc/src/node/interop/interop_server.js" + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "node-stress-server": { + "serverTemplate": "node_server", + "dockerImage": "grpc_stress_node", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "node-stress-client": { + "clientTemplate": "node_client", + "dockerImage": "grpc_stress_node", + "numInstances": 1, + "serverPodSpec": "node-stress-server" + } + } + }, + + "globalSettings": { + "buildDockerImages": false, + "pollIntervalSecs": 10, + "testDurationSecs": 60, + "kubernetesProxyPort": 8001, + "datasetIdNamePrefix": "stress_test_node", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 10 + } +} + diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json index 1dc2d3fe086..30a256de7b3 100644 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json @@ -27,7 +27,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":10 }, "metricsPort": 8081, "metricsArgs": { @@ -39,18 +39,18 @@ "templates": { "cxx_client_opt": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/opt/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/opt/metrics_client" }, "cxx_client_tsan": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/tsan/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/tsan/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/tsan/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/tsan/metrics_client" }, "cxx_client_asan": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/asan/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/asan/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/asan/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/asan/metrics_client" } } }, @@ -68,15 +68,15 @@ "templates": { "cxx_server_opt": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/opt/interop_server" }, "cxx_server_tsan": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/tsan/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/tsan/interop_server" }, "cxx_server_asan": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/asan/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/asan/interop_server" } } }, @@ -124,8 +124,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, + "pollIntervalSecs": 10, + "testDurationSecs": 60, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_opt_tsan", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index b86c0a44210..1cedb922b3e 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":10 }, "metricsPort": 8081, "metricsArgs": { @@ -29,8 +29,8 @@ "templates": { "cxx_client_opt": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/opt/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/opt/metrics_client" } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_opt": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/opt/interop_server" } } }, @@ -74,8 +74,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, + "pollIntervalSecs": 10, + "testDurationSecs": 60, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_opt", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index 182d167cadc..7e913892300 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":10 }, "metricsPort": 8081, "metricsArgs": { @@ -29,8 +29,8 @@ "templates": { "cxx_client_tsan": { "baseTemplate": "default", - "clientImagePath": "/var/local/git/grpc/bins/tsan/stress_test", - "metricsClientImagePath": "/var/local/git/grpc/bins/tsan/metrics_client" + "stressClientCmd": "/var/local/git/grpc/bins/tsan/stress_test", + "metricsClientCmd": "/var/local/git/grpc/bins/tsan/metrics_client" } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_tsan": { "baseTemplate": "default", - "serverImagePath": "/var/local/git/grpc/bins/tsan/interop_server" + "stressServerCmd": "/var/local/git/grpc/bins/tsan/interop_server" } } }, @@ -74,8 +74,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, + "pollIntervalSecs": 10, + "testDurationSecs": 60, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_tsan", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index 9d4ac7ac588..47c9725af32 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -67,12 +67,12 @@ class GlobalSettings: class ClientTemplate: """ Contains all the common settings that are used by a stress client """ - def __init__(self, name, client_image_path, metrics_client_image_path, + def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, wrapper_script_path, poll_interval_secs, client_args_dict, metrics_args_dict): self.name = name - self.client_image_path = client_image_path - self.metrics_client_image_path = metrics_client_image_path + self.stress_client_cmd = stress_client_cmd + self.metrics_client_cmd = metrics_client_cmd self.metrics_port = metrics_port self.wrapper_script_path = wrapper_script_path self.poll_interval_secs = poll_interval_secs @@ -83,10 +83,10 @@ class ClientTemplate: class ServerTemplate: """ Contains all the common settings used by a stress server """ - def __init__(self, name, server_image_path, wrapper_script_path, server_port, + def __init__(self, name, server_cmd, wrapper_script_path, server_port, server_args_dict): self.name = name - self.server_image_path = server_image_path + self.server_cmd = server_cmd self.wrapper_script_path = wrapper_script_path self.server_port = server_port self.server_args_dict = server_args_dict @@ -240,7 +240,7 @@ class Gke: server_env = self.gke_env.copy() server_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'SERVER', - 'STRESS_TEST_IMAGE': server_pod_spec.template.server_image_path, + 'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( server_pod_spec.template.server_args_dict) }) @@ -282,11 +282,11 @@ class Gke: client_env = self.gke_env.copy() client_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', - 'STRESS_TEST_IMAGE': client_pod_spec.template.client_image_path, + 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( client_pod_spec.get_client_args_dict()), - 'METRICS_CLIENT_IMAGE': - client_pod_spec.template.metrics_client_image_path, + 'METRICS_CLIENT_CMD': + client_pod_spec.template.metrics_client_cmd, 'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str( client_pod_spec.template.metrics_args_dict), 'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs) @@ -383,7 +383,7 @@ class Config: for image_name in docker_config_dict.keys(): build_script_path = docker_config_dict[image_name]['buildScript'] dockerfile_dir = docker_config_dict[image_name]['dockerFileDir'] - build_type = docker_config_dict[image_name]['buildType'] + build_type = docker_config_dict[image_name].get('buildType', 'opt') docker_images_dict[image_name] = DockerImage(gcp_project_id, image_name, build_script_path, dockerfile_dir, build_type) @@ -417,8 +417,8 @@ class Config: # Create and add ClientTemplate object to the final client_templates_dict client_templates_dict[template_name] = ClientTemplate( - template_name, temp_dict['clientImagePath'], - temp_dict['metricsClientImagePath'], temp_dict['metricsPort'], + template_name, temp_dict['stressClientCmd'], + temp_dict['metricsClientCmd'], temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), temp_dict['metricsArgs'].copy()) @@ -453,7 +453,7 @@ class Config: # Create and add ServerTemplate object to the final server_templates_dict server_templates_dict[template_name] = ServerTemplate( - template_name, temp_dict['serverImagePath'], + template_name, temp_dict['stressServerCmd'], temp_dict['wrapperScriptPath'], temp_dict['serverPort'], temp_dict['serverArgs'].copy()) From 83a9a6554d2e62de4e9b88d6ea58dd0d04c491c4 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 4 Apr 2016 21:49:15 -0700 Subject: [PATCH 3/7] node wrapper --- tools/gcp/stress_test/run_node.sh | 37 +++++++++++++++++++ tools/run_tests/stress_test/configs/asan.json | 6 +-- tools/run_tests/stress_test/configs/node.json | 22 ++++++++--- .../stress_test/configs/opt-tsan-asan.json | 18 ++++----- tools/run_tests/stress_test/configs/opt.json | 6 +-- tools/run_tests/stress_test/configs/tsan.json | 6 +-- tools/run_tests/stress_test/run_on_gke.py | 25 +++++++------ 7 files changed, 85 insertions(+), 35 deletions(-) create mode 100755 tools/gcp/stress_test/run_node.sh diff --git a/tools/gcp/stress_test/run_node.sh b/tools/gcp/stress_test/run_node.sh new file mode 100755 index 00000000000..329a57e9074 --- /dev/null +++ b/tools/gcp/stress_test/run_node.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This is a wrapper script that was created to help run_server.py and +# run_client.py to launch 'node js' stress clients and stress servers +source ~/.nvm/nvm.sh + +set -ex + +$@ diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index 7414829da84..e17077b7d52 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -29,8 +29,8 @@ "templates": { "cxx_client_asan": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/asan/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/asan/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_asan": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/asan/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] } } }, diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json index 525f9031b32..0c5262a78a9 100644 --- a/tools/run_tests/stress_test/configs/node.json +++ b/tools/run_tests/stress_test/configs/node.json @@ -27,8 +27,16 @@ "templates": { "node_client": { "baseTemplate": "default", - "stressClientCmd": "node /var/local/git/grpc/src/node/stress/stress_client.js", - "metricsClientCmd": "node /var/local/git/grpc/src/node/stress/metrics_client.js" + "stressClientCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", + "node", + "/var/local/git/grpc/src/node/stress/stress_client.js" + ], + "metricsClientCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", + "node", + "/var/local/git/grpc/src/node/stress/metrics_client.js" + ] } } }, @@ -46,7 +54,11 @@ "templates": { "node_server": { "baseTemplate": "default", - "stressServerCmd": "node /var/local/git/grpc/src/node/interop/interop_server.js" + "stressServerCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", + "node", + "/var/local/git/grpc/src/node/interop/interop_server.js" + ] } } }, @@ -64,7 +76,7 @@ "node-stress-client": { "clientTemplate": "node_client", "dockerImage": "grpc_stress_node", - "numInstances": 1, + "numInstances": 2, "serverPodSpec": "node-stress-server" } } @@ -73,7 +85,7 @@ "globalSettings": { "buildDockerImages": false, "pollIntervalSecs": 10, - "testDurationSecs": 60, + "testDurationSecs": 120, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_node", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json index 30a256de7b3..b49683a0147 100644 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json @@ -39,18 +39,18 @@ "templates": { "cxx_client_opt": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/opt/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/opt/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] }, "cxx_client_tsan": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/tsan/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/tsan/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] }, "cxx_client_asan": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/asan/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/asan/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] } } }, @@ -68,15 +68,15 @@ "templates": { "cxx_server_opt": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/opt/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] }, "cxx_server_tsan": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/tsan/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] }, "cxx_server_asan": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/asan/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] } } }, diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index 1cedb922b3e..ae8c00a36dc 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -29,8 +29,8 @@ "templates": { "cxx_client_opt": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/opt/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/opt/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_opt": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/opt/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] } } }, diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index 7e913892300..8d8c954099a 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -29,8 +29,8 @@ "templates": { "cxx_client_tsan": { "baseTemplate": "default", - "stressClientCmd": "/var/local/git/grpc/bins/tsan/stress_test", - "metricsClientCmd": "/var/local/git/grpc/bins/tsan/metrics_client" + "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], + "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] } } }, @@ -48,7 +48,7 @@ "templates": { "cxx_server_tsan": { "baseTemplate": "default", - "stressServerCmd": "/var/local/git/grpc/bins/tsan/interop_server" + "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] } } }, diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index 47c9725af32..50ac05c7b40 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -67,9 +67,9 @@ class GlobalSettings: class ClientTemplate: """ Contains all the common settings that are used by a stress client """ - def __init__(self, name, stress_client_cmd, metrics_client_cmd, - metrics_port, wrapper_script_path, poll_interval_secs, - client_args_dict, metrics_args_dict): + def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, + wrapper_script_path, poll_interval_secs, client_args_dict, + metrics_args_dict): self.name = name self.stress_client_cmd = stress_client_cmd self.metrics_client_cmd = metrics_client_cmd @@ -285,8 +285,7 @@ class Gke: 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( client_pod_spec.get_client_args_dict()), - 'METRICS_CLIENT_CMD': - client_pod_spec.template.metrics_client_cmd, + 'METRICS_CLIENT_CMD': client_pod_spec.template.metrics_client_cmd, 'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str( client_pod_spec.template.metrics_args_dict), 'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs) @@ -416,11 +415,13 @@ class Config: temp_dict.update(templates_dict[template_name]) # Create and add ClientTemplate object to the final client_templates_dict + stress_client_cmd = ' '.join(temp_dict['stressClientCmd']) + metrics_client_cmd = ' '.join(temp_dict['metricsClientCmd']) client_templates_dict[template_name] = ClientTemplate( - template_name, temp_dict['stressClientCmd'], - temp_dict['metricsClientCmd'], temp_dict['metricsPort'], - temp_dict['wrapperScriptPath'], temp_dict['pollIntervalSecs'], - temp_dict['clientArgs'].copy(), temp_dict['metricsArgs'].copy()) + template_name, stress_client_cmd, metrics_client_cmd, + temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], + temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), + temp_dict['metricsArgs'].copy()) return client_templates_dict @@ -452,10 +453,10 @@ class Config: temp_dict.update(templates_dict[template_name]) # Create and add ServerTemplate object to the final server_templates_dict + stress_server_cmd = ' '.join(temp_dict['stressServerCmd']) server_templates_dict[template_name] = ServerTemplate( - template_name, temp_dict['stressServerCmd'], - temp_dict['wrapperScriptPath'], temp_dict['serverPort'], - temp_dict['serverArgs'].copy()) + template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], + temp_dict['serverPort'], temp_dict['serverArgs'].copy()) return server_templates_dict From 73b6a3b63bc40f836b068c4ac3b8cc45b98f7bbc Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 4 Apr 2016 22:38:53 -0700 Subject: [PATCH 4/7] Config file settings --- tools/run_tests/stress_test/configs/asan.json | 10 +- .../stress_test/configs/node-cxx.json | 97 +++++++++++++++++++ tools/run_tests/stress_test/configs/node.json | 12 +-- .../stress_test/configs/opt-tsan-asan.json | 14 +-- tools/run_tests/stress_test/configs/opt.json | 6 +- tools/run_tests/stress_test/configs/tsan.json | 10 +- 6 files changed, 123 insertions(+), 26 deletions(-) create mode 100644 tools/run_tests/stress_test/configs/node-cxx.json diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index e17077b7d52..768088d93dd 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":10 + "metrics_collection_interval_secs":60 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-asan": { "clientTemplate": "cxx_client_asan", "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 10, + "numInstances": 20, "serverPodSpec": "stress-server-asan" } } @@ -74,9 +74,9 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 10, - "testDurationSecs": 60, - "kubernetesProxyPort": 8001, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8003, "datasetIdNamePrefix": "stress_test_asan", "summaryTableId": "summary", "qpsTableId": "qps", diff --git a/tools/run_tests/stress_test/configs/node-cxx.json b/tools/run_tests/stress_test/configs/node-cxx.json new file mode 100644 index 00000000000..c4245bf9df2 --- /dev/null +++ b/tools/run_tests/stress_test/configs/node-cxx.json @@ -0,0 +1,97 @@ +{ + "dockerImages": { + "grpc_stress_cxx_opt" : { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_cxx", + "buildType": "opt" + }, + "grpc_stress_node": { + "buildScript": "tools/jenkins/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_node" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + } + } + }, + "templates": { + "node_client": { + "baseTemplate": "default", + "stressClientCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", + "node", + "/var/local/git/grpc/src/node/stress/stress_client.js" + ], + "metricsClientCmd": [ + "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", + "node", + "/var/local/git/grpc/src/node/stress/metrics_client.js" + ] + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + } + } + }, + "templates": { + "cxx_server_opt": { + "baseTemplate": "default", + "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "stress-server-cxx-opt": { + "serverTemplate": "cxx_server_opt", + "dockerImage": "grpc_stress_cxx_opt", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "stress-client-node": { + "clientTemplate": "node_client", + "dockerImage": "grpc_stress_node", + "numInstances": 20, + "serverPodSpec": "stress-server-cxx-opt" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8006, + "datasetIdNamePrefix": "stress_test_node_cxx_opt", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json index 0c5262a78a9..7a48c56a5e2 100644 --- a/tools/run_tests/stress_test/configs/node.json +++ b/tools/run_tests/stress_test/configs/node.json @@ -76,21 +76,21 @@ "node-stress-client": { "clientTemplate": "node_client", "dockerImage": "grpc_stress_node", - "numInstances": 2, + "numInstances": 15, "serverPodSpec": "node-stress-server" } } }, "globalSettings": { - "buildDockerImages": false, - "pollIntervalSecs": 10, - "testDurationSecs": 120, - "kubernetesProxyPort": 8001, + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8005, "datasetIdNamePrefix": "stress_test_node", "summaryTableId": "summary", "qpsTableId": "qps", - "podWarmupSecs": 10 + "podWarmupSecs": 60 } } diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json index b49683a0147..4f172ef30bc 100644 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json @@ -27,7 +27,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":10 + "metrics_collection_interval_secs": 60 }, "metricsPort": 8081, "metricsArgs": { @@ -104,19 +104,19 @@ "stress-client-opt": { "clientTemplate": "cxx_client_opt", "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 3, + "numInstances": 5, "serverPodSpec": "stress-server-opt" }, "stress-client-tsan": { "clientTemplate": "cxx_client_tsan", "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 3, + "numInstances": 10, "serverPodSpec": "stress-server-tsan" }, "stress-client-asan": { "clientTemplate": "cxx_client_asan", "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 3, + "numInstances": 10, "serverPodSpec": "stress-server-asan" } } @@ -124,9 +124,9 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 10, - "testDurationSecs": 60, - "kubernetesProxyPort": 8001, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8004, "datasetIdNamePrefix": "stress_test_opt_tsan", "summaryTableId": "summary", "qpsTableId": "qps", diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index ae8c00a36dc..ffd4a704c34 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":10 + "metrics_collection_interval_secs": 60 }, "metricsPort": 8081, "metricsArgs": { @@ -74,8 +74,8 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 10, - "testDurationSecs": 60, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, "kubernetesProxyPort": 8001, "datasetIdNamePrefix": "stress_test_opt", "summaryTableId": "summary", diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index 8d8c954099a..f8d3f878e16 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -17,7 +17,7 @@ "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":10 + "metrics_collection_interval_secs":60 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-tsan": { "clientTemplate": "cxx_client_tsan", "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 10, + "numInstances": 20, "serverPodSpec": "stress-server-tsan" } } @@ -74,9 +74,9 @@ "globalSettings": { "buildDockerImages": true, - "pollIntervalSecs": 10, - "testDurationSecs": 60, - "kubernetesProxyPort": 8001, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8002, "datasetIdNamePrefix": "stress_test_tsan", "summaryTableId": "summary", "qpsTableId": "qps", From aecac88e02bd3b7ae3af80c471f73ede42618de2 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 4 Apr 2016 22:48:15 -0700 Subject: [PATCH 5/7] fix copyrights --- .../stress_test/grpc_interop_stress_cxx/Dockerfile.template | 2 +- .../stress_test/grpc_interop_stress_node/Dockerfile.template | 2 +- tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile | 2 +- .../dockerfile/stress_test/grpc_interop_stress_node/Dockerfile | 2 +- tools/gcp/stress_test/run_client.py | 2 +- tools/gcp/stress_test/run_node.sh | 2 +- tools/gcp/stress_test/run_server.py | 2 +- tools/run_tests/stress_test/run_on_gke.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template index 61b16c77159..ac087c5da74 100644 --- a/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile.template @@ -1,6 +1,6 @@ %YAML 1.2 --- | - # Copyright 2015, Google Inc. + # Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template index 6520ec29c1d..c50d38d1eca 100644 --- a/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile.template @@ -1,6 +1,6 @@ %YAML 1.2 --- | - # Copyright 2015, Google Inc. + # Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile index 97dc5ca77de..556a26ee139 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile index a932b4087d8..f70add4e31a 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/gcp/stress_test/run_client.py b/tools/gcp/stress_test/run_client.py index 71ee0c7c3cc..9a4bc8a3918 100755 --- a/tools/gcp/stress_test/run_client.py +++ b/tools/gcp/stress_test/run_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/gcp/stress_test/run_node.sh b/tools/gcp/stress_test/run_node.sh index 329a57e9074..4a4da6fc8b8 100755 --- a/tools/gcp/stress_test/run_node.sh +++ b/tools/gcp/stress_test/run_node.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/gcp/stress_test/run_server.py b/tools/gcp/stress_test/run_server.py index a02edf8ea41..0d9a653d186 100755 --- a/tools/gcp/stress_test/run_server.py +++ b/tools/gcp/stress_test/run_server.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index 50ac05c7b40..1485281e794 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2.7 -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without From 9f0e3601c7e28e20716e560d3ca8226853121354 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 5 Apr 2016 16:21:44 -0700 Subject: [PATCH 6/7] Some debug statements and reformatting --- tools/gcp/stress_test/stress_test_utils.py | 35 +++++++++++++++------- tools/run_tests/stress_test/run_on_gke.py | 2 ++ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/tools/gcp/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py index 79d9dea675c..6c7fe44dc12 100755 --- a/tools/gcp/stress_test/stress_test_utils.py +++ b/tools/gcp/stress_test/stress_test_utils.py @@ -107,12 +107,22 @@ class BigQueryHelper: query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND ' 'event_type="%s"') % (self.dataset_id, self.summary_table_id, self.run_id, EventType.FAILURE) - query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) - page = self.bq.jobs().getQueryResults(**query_job['jobReference']).execute( - num_retries=num_query_retries) - num_failures = int(page['totalRows']) - print 'num rows: ', num_failures - return num_failures > 0 + try: + query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) + page = self.bq.jobs().getQueryResults( + **query_job['jobReference']).execute(num_retries=num_query_retries) + num_failures = int(page['totalRows']) + print 'num rows: ', num_failures + return num_failures > 0 + # TODO (sreek): Cleanup the following lines once we have a better idea of + # why we sometimes get KeyError exceptions in long running test cases + except KeyError: + print 'KeyError in check_if_any_tests_failed()' + print 'Query:', query + print 'Query result page:', page + except: + print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info() + print 'Query: ', query def print_summary_records(self, num_query_retries=3): line = '-' * 120 @@ -126,8 +136,9 @@ class BigQueryHelper: self.dataset_id, self.summary_table_id, self.run_id) query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) - print '{:<25} {:<12} {:<12} {:<30} {}'.format( - 'Pod name', 'Image type', 'Event type', 'Date', 'Details') + print '{:<25} {:<12} {:<12} {:<30} {}'.format('Pod name', 'Image type', + 'Event type', 'Date', + 'Details') print line page_token = None while True: @@ -136,9 +147,11 @@ class BigQueryHelper: **query_job['jobReference']).execute(num_retries=num_query_retries) rows = page.get('rows', []) for row in rows: - print '{:<25} {:<12} {:<12} {:<30} {}'.format( - row['f'][0]['v'], row['f'][1]['v'], row['f'][2]['v'], - row['f'][3]['v'], row['f'][4]['v']) + print '{:<25} {:<12} {:<12} {:<30} {}'.format(row['f'][0]['v'], + row['f'][1]['v'], + row['f'][2]['v'], + row['f'][3]['v'], + row['f'][4]['v']) page_token = page.get('pageToken') if not page_token: break diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index 1485281e794..db3ba28346d 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -530,6 +530,8 @@ def run_tests(config): # run id. This is useful in debugging when looking at records in Biq query) run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) + print 'Run id:', run_id + print 'Dataset id:', dataset_id bq_helper = BigQueryHelper(run_id, '', '', config.global_settings.gcp_project_id, dataset_id, From 79e3f09e1ae7c4eb313c4c8fbdb62a0f27f58e1f Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 5 Apr 2016 16:51:05 -0700 Subject: [PATCH 7/7] Code review comments --- .../grpc_interop_stress_node/build_interop_stress.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh index 4d4290d0b41..b99fd444eea 100755 --- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh +++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh @@ -41,8 +41,6 @@ cd /var/local/git/grpc nvm use 0.12 nvm alias default 0.12 # prevent the need to run 'nvm use' in every shell -make install-certs - # build Node interop client & server npm install -g node-gyp npm install --unsafe-perm --build-from-source