From 1b38bb4bc745cf9f7ee18728d96698d3652dad9d Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 14 Dec 2015 17:22:38 -0800
Subject: [PATCH 1/8] initial version of run stress tests script

---
 .../jenkins/grpc_interop_cxx/build_interop.sh |   2 +-
 tools/run_tests/run_stress_tests.py           | 378 ++++++++++++++++++
 2 files changed, 379 insertions(+), 1 deletion(-)
 create mode 100755 tools/run_tests/run_stress_tests.py

diff --git a/tools/jenkins/grpc_interop_cxx/build_interop.sh b/tools/jenkins/grpc_interop_cxx/build_interop.sh
index 1c0828d23a6..3634b7123f3 100755
--- a/tools/jenkins/grpc_interop_cxx/build_interop.sh
+++ b/tools/jenkins/grpc_interop_cxx/build_interop.sh
@@ -42,4 +42,4 @@ cd /var/local/git/grpc
 make install-certs
 
 # build C++ interop client & server
-make interop_client interop_server
+make interop_client interop_server stress_test
diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
new file mode 100755
index 00000000000..529dc4a48de
--- /dev/null
+++ b/tools/run_tests/run_stress_tests.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""Run stress test in C++"""
+
+import argparse
+import atexit
+import dockerjob
+import itertools
+import jobset
+import json
+import multiprocessing
+import os
+import re
+import report_utils
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+
+# Docker doesn't clean up after itself, so we do it on exit.
+atexit.register(lambda: subprocess.call(['stty', 'echo']))
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+os.chdir(ROOT)
+
+_DEFAULT_SERVER_PORT = 8080
+_DEFAULT_METRICS_PORT = 8081
+_DEFAULT_TEST_CASES = 'empty_unary:20,large_unary:20,client_streaming:20,server_streaming:20,empty_stream:20'
+_DEFAULT_NUM_CHANNELS_PER_SERVER = 5
+_DEFAULT_NUM_STUBS_PER_CHANNEL = 10
+
+# 15 mins default
+#_DEFAULT_TEST_DURATION_SECS = 900
+_DEFAULT_TEST_DURATION_SECS = 10
+
+class CXXLanguage:
+
+  def __init__(self):
+    self.client_cwd = None
+    self.server_cwd = None
+    self.safename = 'cxx'
+
+  def client_cmd(self, args):
+    return ['bins/opt/stress_test'] + args
+
+  def server_cmd(self, args):
+    return ['bins/opt/interop_server'] + args
+
+  def global_env(self):
+    return {}
+
+  def __str__(self):
+    return 'c++'
+
+
+_LANGUAGES = {'c++': CXXLanguage(),}
+
+# languages supported as cloud_to_cloud servers
+_SERVERS = ['c++']
+
+DOCKER_WORKDIR_ROOT = '/var/local/git/grpc'
+
+
+def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
+  """Wraps given cmdline array to create 'docker run' cmdline from it."""
+  docker_cmdline = ['docker', 'run', '-i', '--rm=true']
+
+  # turn environ into -e docker args
+  if environ:
+    for k, v in environ.iteritems():
+      docker_cmdline += ['-e', '%s=%s' % (k, v)]
+
+  # set working directory
+  workdir = DOCKER_WORKDIR_ROOT
+  if cwd:
+    workdir = os.path.join(workdir, cwd)
+  docker_cmdline += ['-w', workdir]
+
+  docker_cmdline += docker_args + [image] + cmdline
+  return docker_cmdline
+
+
+def bash_login_cmdline(cmdline):
+  """Creates bash -l -c cmdline from args list."""
+  # Use login shell:
+  # * rvm and nvm require it
+  # * makes error messages clearer if executables are missing
+  return ['bash', '-l', '-c', ' '.join(cmdline)]
+
+
+def _job_kill_handler(job):
+  if job._spec.container_name:
+    dockerjob.docker_kill(job._spec.container_name)
+    # When the job times out and we decide to kill it,
+    # we need to wait a before restarting the job
+    # to prevent "container name already in use" error.
+    # TODO(jtattermusch): figure out a cleaner way to to this.
+    time.sleep(2)
+
+
+def cloud_to_cloud_jobspec(language,
+                           test_cases,
+                           server_addresses,
+                           test_duration_secs,
+                           num_channels_per_server,
+                           num_stubs_per_channel,
+                           metrics_port,
+                           docker_image=None):
+  """Creates jobspec for cloud-to-cloud interop test"""
+  cmdline = bash_login_cmdline(language.client_cmd([
+      '--test_cases=%s' % test_cases, '--server_addresses=%s' %
+      server_addresses, '--test_duration_secs=%s' % test_duration_secs,
+      '--num_stubs_per_channel=%s' % num_stubs_per_channel,
+      '--num_channels_per_server=%s' % num_channels_per_server,
+      '--metrics_port=%s' % metrics_port
+  ]))
+  print cmdline
+  cwd = language.client_cwd
+  environ = language.global_env()
+  if docker_image:
+    container_name = dockerjob.random_name('interop_client_%s' %
+                                           language.safename)
+    cmdline = docker_run_cmdline(
+        cmdline,
+        image=docker_image,
+        environ=environ,
+        cwd=cwd,
+        docker_args=['--net=host', '--name', container_name])
+    cwd = None
+
+  test_job = jobset.JobSpec(cmdline=cmdline,
+                            cwd=cwd,
+                            environ=environ,
+                            shortname='cloud_to_cloud:%s:%s_server:stress_test' % (
+                                language, server_name),
+                            timeout_seconds=test_duration_secs * 2,
+                            flake_retries=5 if args.allow_flakes else 0,
+                            timeout_retries=2 if args.allow_flakes else 0,
+                            kill_handler=_job_kill_handler)
+  test_job.container_name = container_name
+  return test_job
+
+
+def server_jobspec(language, docker_image, test_duration_secs):
+  """Create jobspec for running a server"""
+  container_name = dockerjob.random_name('interop_server_%s' %
+                                         language.safename)
+  cmdline = bash_login_cmdline(language.server_cmd(['--port=%s' %
+                                                    _DEFAULT_SERVER_PORT]))
+  environ = language.global_env()
+  docker_cmdline = docker_run_cmdline(
+      cmdline,
+      image=docker_image,
+      cwd=language.server_cwd,
+      environ=environ,
+      docker_args=['-p', str(_DEFAULT_SERVER_PORT), '--name', container_name])
+
+  server_job = jobset.JobSpec(cmdline=docker_cmdline,
+                              environ=environ,
+                              shortname='interop_server_%s' % language,
+                              timeout_seconds=test_duration_secs * 3)
+  server_job.container_name = container_name
+  return server_job
+
+
+def build_interop_image_jobspec(language, tag=None):
+  """Creates jobspec for building stress test docker image for a language"""
+  if not tag:
+    tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4())
+  env = {'INTEROP_IMAGE': tag,
+         'BASE_NAME': 'grpc_interop_%s' % language.safename}
+  env['TTY_FLAG'] = '-t'
+  build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_image.sh'],
+                             environ=env,
+                             shortname='build_docker_%s' % (language),
+                             timeout_seconds=30 * 60)
+  build_job.tag = tag
+  return build_job
+
+
+def aggregate_http2_results(stdout):
+  match = re.search(r'\{"cases[^\]]*\]\}', stdout)
+  if not match:
+    return None
+
+  results = json.loads(match.group(0))
+  skipped = 0
+  passed = 0
+  failed = 0
+  failed_cases = []
+  for case in results['cases']:
+    if case.get('skipped', False):
+      skipped += 1
+    else:
+      if case.get('passed', False):
+        passed += 1
+      else:
+        failed += 1
+        failed_cases.append(case.get('name', 'NONAME'))
+  return {
+      'passed': passed,
+      'failed': failed,
+      'skipped': skipped,
+      'failed_cases': ', '.join(failed_cases),
+      'percent': 1.0 * passed / (passed + failed)
+  }
+
+
+argp = argparse.ArgumentParser(description='Run stress tests.')
+argp.add_argument('-l',
+                  '--language',
+                  choices=['all'] + sorted(_LANGUAGES),
+                  nargs='+',
+                  default=['all'],
+                  help='Clients to run.')
+argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
+argp.add_argument(
+    '-s',
+    '--server',
+    choices=['all'] + sorted(_SERVERS),
+    action='append',
+    help='Run cloud_to_cloud servers in a separate docker ' + 'image.',
+    default=[])
+argp.add_argument(
+    '--override_server',
+    action='append',
+    type=lambda kv: kv.split('='),
+    help=
+    'Use servername=HOST:PORT to explicitly specify a server. E.g. '
+    'csharp=localhost:50000',
+    default=[])
+argp.add_argument('--test_duration_secs',
+                  action='append',
+                  help='The duration of the test in seconds',
+                  default=[_DEFAULT_TEST_DURATION_SECS])
+argp.add_argument(
+    '--allow_flakes',
+    default=False,
+    action='store_const',
+    const=True,
+    help=
+    'Allow flaky tests to show as passing (re-runs failed tests up to five times)')
+
+args = argp.parse_args()
+
+servers = set(
+    s
+    for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x]
+                                           for x in args.server))
+
+languages = set(_LANGUAGES[l]
+                for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys(
+                ) if x == 'all' else [x] for x in args.language))
+
+docker_images = {}
+# languages for which to build docker images
+languages_to_build = set(
+    _LANGUAGES[k]
+    for k in set([str(l) for l in languages] + [s for s in servers]))
+build_jobs = []
+for l in languages_to_build:
+  job = build_interop_image_jobspec(l)
+  docker_images[str(l)] = job.tag
+  build_jobs.append(job)
+
+if build_jobs:
+  jobset.message('START', 'Building interop docker images.', do_newline=True)
+  num_failures, _ = jobset.run(build_jobs,
+                               newline_on_success=True,
+                               maxjobs=args.jobs)
+  if num_failures == 0:
+    jobset.message('SUCCESS',
+                   'All docker images built successfully.',
+                   do_newline=True)
+  else:
+    jobset.message('FAILED',
+                   'Failed to build interop docker images.',
+                   do_newline=True)
+    for image in docker_images.itervalues():
+      dockerjob.remove_image(image, skip_nonexistent=True)
+    sys.exit(1)
+
+# Start interop servers.
+server_jobs = {}
+server_addresses = {}
+try:
+  for s in servers:
+    lang = str(s)
+    spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), _DEFAULT_TEST_DURATION_SECS)
+    job = dockerjob.DockerJob(spec)
+    server_jobs[lang] = job
+    server_addresses[lang] = ('localhost',
+                              job.mapped_port(_DEFAULT_SERVER_PORT))
+
+  jobs = []
+
+  for server in args.override_server:
+    server_name = server[0]
+    (server_host, server_port) = server[1].split(':')
+    server_addresses[server_name] = (server_host, server_port)
+
+  for server_name, server_address in server_addresses.iteritems():
+    (server_host, server_port) = server_address
+    for language in languages:
+      test_job = cloud_to_cloud_jobspec(
+          language,
+          _DEFAULT_TEST_CASES,
+          ('%s:%s' % (server_host, server_port)),
+          _DEFAULT_TEST_DURATION_SECS,
+          _DEFAULT_NUM_CHANNELS_PER_SERVER,
+          _DEFAULT_NUM_STUBS_PER_CHANNEL,
+          _DEFAULT_METRICS_PORT,
+          docker_image=docker_images.get(str(language)))
+      jobs.append(test_job)
+
+  if not jobs:
+    print 'No jobs to run.'
+    for image in docker_images.itervalues():
+      dockerjob.remove_image(image, skip_nonexistent=True)
+    sys.exit(1)
+
+  num_failures, resultset = jobset.run(jobs,
+                                       newline_on_success=True,
+                                       maxjobs=args.jobs)
+  if num_failures:
+    jobset.message('FAILED', 'Some tests failed', do_newline=True)
+  else:
+    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
+
+  report_utils.render_junit_xml_report(resultset, 'report.xml')
+
+  for name, job in resultset.iteritems():
+    if "http2" in name:
+      job[0].http2results = aggregate_http2_results(job[0].message)
+
+  report_utils.render_interop_html_report(
+      set([str(l) for l in languages]), servers, [], [], [], resultset,
+      num_failures, 0, 0)
+
+finally:
+  # Check if servers are still running.
+  for server, job in server_jobs.iteritems():
+    if not job.is_running():
+      print 'Server "%s" has exited prematurely.' % server
+
+  dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
+
+  for image in docker_images.itervalues():
+    print 'Removing docker image %s' % image
+    dockerjob.remove_image(image)

From cc16931dea6b70ea67a6cc2549a3e535fbc3e577 Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 21 Dec 2015 01:12:49 -0800
Subject: [PATCH 2/8] Do not wait for metrics server. This will allow stress
 tests to terminate

---
 test/cpp/interop/stress_test.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/cpp/interop/stress_test.cc b/test/cpp/interop/stress_test.cc
index 934f4f5722d..e2e73cf4b13 100644
--- a/test/cpp/interop/stress_test.cc
+++ b/test/cpp/interop/stress_test.cc
@@ -270,6 +270,5 @@ int main(int argc, char** argv) {
     it->join();
   }
 
-  metrics_server->Wait();
   return 0;
 }

From ca8e3d77877b63d9bb6a675c9c6c122d72f47e7c Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 09:39:05 -0800
Subject: [PATCH 3/8] Modify run_tests to run stress tests

---
 tools/run_tests/run_stress_tests.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index 529dc4a48de..d857fdb1102 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -59,7 +59,7 @@ _DEFAULT_NUM_STUBS_PER_CHANNEL = 10
 
 # 15 mins default
 #_DEFAULT_TEST_DURATION_SECS = 900
-_DEFAULT_TEST_DURATION_SECS = 10
+_DEFAULT_TEST_DURATION_SECS = 30
 
 class CXXLanguage:
 
@@ -258,9 +258,8 @@ argp.add_argument(
     'csharp=localhost:50000',
     default=[])
 argp.add_argument('--test_duration_secs',
-                  action='append',
                   help='The duration of the test in seconds',
-                  default=[_DEFAULT_TEST_DURATION_SECS])
+                  default=_DEFAULT_TEST_DURATION_SECS)
 argp.add_argument(
     '--allow_flakes',
     default=False,
@@ -314,7 +313,7 @@ server_addresses = {}
 try:
   for s in servers:
     lang = str(s)
-    spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), _DEFAULT_TEST_DURATION_SECS)
+    spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), args.test_duration_secs)
     job = dockerjob.DockerJob(spec)
     server_jobs[lang] = job
     server_addresses[lang] = ('localhost',
@@ -334,7 +333,7 @@ try:
           language,
           _DEFAULT_TEST_CASES,
           ('%s:%s' % (server_host, server_port)),
-          _DEFAULT_TEST_DURATION_SECS,
+          args.test_duration_secs,
           _DEFAULT_NUM_CHANNELS_PER_SERVER,
           _DEFAULT_NUM_STUBS_PER_CHANNEL,
           _DEFAULT_METRICS_PORT,

From e371742d832fad56d03ab62f8b36ebbc219af29a Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 10:14:38 -0800
Subject: [PATCH 4/8] Separate interop images and interop_stress images

---
 tools/jenkins/build_interop_stress_image.sh   | 86 +++++++++++++++++++
 .../jenkins/grpc_interop_cxx/build_interop.sh |  2 +-
 .../grpc_interop_stress_cxx/Dockerfile        | 75 ++++++++++++++++
 .../build_interop_stress.sh                   | 45 ++++++++++
 tools/jenkins/run_interop_stress.sh           | 37 ++++++++
 tools/run_tests/run_stress_tests.py           | 14 +--
 6 files changed, 251 insertions(+), 8 deletions(-)
 create mode 100755 tools/jenkins/build_interop_stress_image.sh
 create mode 100644 tools/jenkins/grpc_interop_stress_cxx/Dockerfile
 create mode 100755 tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh
 create mode 100755 tools/jenkins/run_interop_stress.sh

diff --git a/tools/jenkins/build_interop_stress_image.sh b/tools/jenkins/build_interop_stress_image.sh
new file mode 100755
index 00000000000..6b22dce6c02
--- /dev/null
+++ b/tools/jenkins/build_interop_stress_image.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This script is invoked by run_interop_tests.py to build the docker image
+# for interop testing. You should never need to call this script on your own.
+
+set -x
+
+# Params:
+#  INTEROP_IMAGE - name of tag of the final interop image
+#  BASE_NAME - base name used to locate the base Dockerfile and build script
+#  TTY_FLAG - optional -t flag to make docker allocate tty
+#  BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the
+#    docker run command
+
+cd `dirname $0`/../..
+GRPC_ROOT=`pwd`
+MOUNT_ARGS="-v $GRPC_ROOT:/var/local/jenkins/grpc:ro"
+
+mkdir -p /tmp/ccache
+
+# Mount service account dir if available.
+# If service_directory does not contain the service account JSON file,
+# some of the tests will fail.
+if [ -e $HOME/service_account ]
+then
+  MOUNT_ARGS+=" -v $HOME/service_account:/var/local/jenkins/service_account:ro"
+fi
+
+# Use image name based on Dockerfile checksum
+BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/jenkins/$BASE_NAME/Dockerfile | cut -f1 -d\ `
+
+# Make sure base docker image has been built. Should be instantaneous if so.
+docker build -t $BASE_IMAGE --force-rm=true tools/jenkins/$BASE_NAME || exit $?
+
+# Create a local branch so the child Docker script won't complain
+git branch -f jenkins-docker
+
+CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)"
+
+# Prepare image for interop tests, commit it on success.
+(docker run \
+  -e CCACHE_DIR=/tmp/ccache \
+  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -i $TTY_FLAG \
+  $MOUNT_ARGS \
+  $BUILD_INTEROP_DOCKER_EXTRA_ARGS \
+  -v /tmp/ccache:/tmp/ccache \
+  --name=$CONTAINER_NAME \
+  $BASE_IMAGE \
+  bash -l /var/local/jenkins/grpc/tools/jenkins/$BASE_NAME/build_interop_stress.sh \
+  && docker commit $CONTAINER_NAME $INTEROP_IMAGE \
+  && echo "Successfully built image $INTEROP_IMAGE")
+EXITCODE=$?
+
+# remove intermediate container, possibly killing it first
+docker rm -f $CONTAINER_NAME
+
+exit $EXITCODE
diff --git a/tools/jenkins/grpc_interop_cxx/build_interop.sh b/tools/jenkins/grpc_interop_cxx/build_interop.sh
index 3634b7123f3..1c0828d23a6 100755
--- a/tools/jenkins/grpc_interop_cxx/build_interop.sh
+++ b/tools/jenkins/grpc_interop_cxx/build_interop.sh
@@ -42,4 +42,4 @@ cd /var/local/git/grpc
 make install-certs
 
 # build C++ interop client & server
-make interop_client interop_server stress_test
+make interop_client interop_server
diff --git a/tools/jenkins/grpc_interop_stress_cxx/Dockerfile b/tools/jenkins/grpc_interop_stress_cxx/Dockerfile
new file mode 100644
index 00000000000..1fa19075330
--- /dev/null
+++ b/tools/jenkins/grpc_interop_stress_cxx/Dockerfile
@@ -0,0 +1,75 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# A work-in-progress Dockerfile that allows running gRPC test suites
+# inside a docker container.
+
+FROM debian:jessie
+
+# Install Git.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  gyp \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  strace \
+  python-dev \
+  python-setuptools \
+  python-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+##################
+# C++ dependencies
+RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang
+
+# Define the default command.
+CMD ["bash"]
diff --git a/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh
new file mode 100755
index 00000000000..01f9a9c02e7
--- /dev/null
+++ b/tools/jenkins/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds C++ interop server and client in a base image.
+set -e
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+cd /var/local/git/grpc
+
+make install-certs
+
+# build C++ interop stress client, interop client and server
+make stress_test interop_client interop_server
diff --git a/tools/jenkins/run_interop_stress.sh b/tools/jenkins/run_interop_stress.sh
new file mode 100755
index 00000000000..22d81db8bcd
--- /dev/null
+++ b/tools/jenkins/run_interop_stress.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This script is invoked by Jenkins and runs interop test suite.
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../..
+
+tools/run_tests/run_stress_tests.py -l all -s all -j 12 $@ || true
diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index d857fdb1102..406c6626336 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -162,8 +162,8 @@ def cloud_to_cloud_jobspec(language,
                             shortname='cloud_to_cloud:%s:%s_server:stress_test' % (
                                 language, server_name),
                             timeout_seconds=test_duration_secs * 2,
-                            flake_retries=5 if args.allow_flakes else 0,
-                            timeout_retries=2 if args.allow_flakes else 0,
+                            flake_retries=0,
+                            timeout_retries=0,
                             kill_handler=_job_kill_handler)
   test_job.container_name = container_name
   return test_job
@@ -191,14 +191,14 @@ def server_jobspec(language, docker_image, test_duration_secs):
   return server_job
 
 
-def build_interop_image_jobspec(language, tag=None):
+def build_interop_stress_image_jobspec(language, tag=None):
   """Creates jobspec for building stress test docker image for a language"""
   if not tag:
-    tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4())
+    tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4())
   env = {'INTEROP_IMAGE': tag,
-         'BASE_NAME': 'grpc_interop_%s' % language.safename}
+         'BASE_NAME': 'grpc_interop_stress_%s' % language.safename}
   env['TTY_FLAG'] = '-t'
-  build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_image.sh'],
+  build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_stress_image.sh'],
                              environ=env,
                              shortname='build_docker_%s' % (language),
                              timeout_seconds=30 * 60)
@@ -286,7 +286,7 @@ languages_to_build = set(
     for k in set([str(l) for l in languages] + [s for s in servers]))
 build_jobs = []
 for l in languages_to_build:
-  job = build_interop_image_jobspec(l)
+  job = build_interop_stress_image_jobspec(l)
   docker_images[str(l)] = job.tag
   build_jobs.append(job)
 

From 7b89b9719807f168a313308972e77a1c278d3ea8 Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 13:06:57 -0800
Subject: [PATCH 5/8] disable -t flag

---
 tools/run_tests/run_stress_tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index 406c6626336..f89312fcca8 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -197,7 +197,7 @@ def build_interop_stress_image_jobspec(language, tag=None):
     tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4())
   env = {'INTEROP_IMAGE': tag,
          'BASE_NAME': 'grpc_interop_stress_%s' % language.safename}
-  env['TTY_FLAG'] = '-t'
+  #env['TTY_FLAG'] = '-t'
   build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_stress_image.sh'],
                              environ=env,
                              shortname='build_docker_%s' % (language),

From 66977607c1b7f1d716d18902166c9977ac9fa628 Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 14:34:11 -0800
Subject: [PATCH 6/8] adjust default test duration

---
 tools/run_tests/run_stress_tests.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index f89312fcca8..0e2c883f23a 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -58,8 +58,7 @@ _DEFAULT_NUM_CHANNELS_PER_SERVER = 5
 _DEFAULT_NUM_STUBS_PER_CHANNEL = 10
 
 # 15 mins default
-#_DEFAULT_TEST_DURATION_SECS = 900
-_DEFAULT_TEST_DURATION_SECS = 30
+_DEFAULT_TEST_DURATION_SECS = 900
 
 class CXXLanguage:
 
@@ -197,7 +196,6 @@ def build_interop_stress_image_jobspec(language, tag=None):
     tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4())
   env = {'INTEROP_IMAGE': tag,
          'BASE_NAME': 'grpc_interop_stress_%s' % language.safename}
-  #env['TTY_FLAG'] = '-t'
   build_job = jobset.JobSpec(cmdline=['tools/jenkins/build_interop_stress_image.sh'],
                              environ=env,
                              shortname='build_docker_%s' % (language),

From 51571a839b4ae51fe37e203cdec29d0c7f02b23c Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 15:33:35 -0800
Subject: [PATCH 7/8] remove code not relevant for stress test results

---
 tools/run_tests/run_stress_tests.py | 40 -----------------------------
 1 file changed, 40 deletions(-)

diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index 0e2c883f23a..0dc38986ee9 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -38,7 +38,6 @@ import json
 import multiprocessing
 import os
 import re
-import report_utils
 import subprocess
 import sys
 import tempfile
@@ -203,35 +202,6 @@ def build_interop_stress_image_jobspec(language, tag=None):
   build_job.tag = tag
   return build_job
 
-
-def aggregate_http2_results(stdout):
-  match = re.search(r'\{"cases[^\]]*\]\}', stdout)
-  if not match:
-    return None
-
-  results = json.loads(match.group(0))
-  skipped = 0
-  passed = 0
-  failed = 0
-  failed_cases = []
-  for case in results['cases']:
-    if case.get('skipped', False):
-      skipped += 1
-    else:
-      if case.get('passed', False):
-        passed += 1
-      else:
-        failed += 1
-        failed_cases.append(case.get('name', 'NONAME'))
-  return {
-      'passed': passed,
-      'failed': failed,
-      'skipped': skipped,
-      'failed_cases': ', '.join(failed_cases),
-      'percent': 1.0 * passed / (passed + failed)
-  }
-
-
 argp = argparse.ArgumentParser(description='Run stress tests.')
 argp.add_argument('-l',
                   '--language',
@@ -352,16 +322,6 @@ try:
   else:
     jobset.message('SUCCESS', 'All tests passed', do_newline=True)
 
-  report_utils.render_junit_xml_report(resultset, 'report.xml')
-
-  for name, job in resultset.iteritems():
-    if "http2" in name:
-      job[0].http2results = aggregate_http2_results(job[0].message)
-
-  report_utils.render_interop_html_report(
-      set([str(l) for l in languages]), servers, [], [], [], resultset,
-      num_failures, 0, 0)
-
 finally:
   # Check if servers are still running.
   for server, job in server_jobs.iteritems():

From d609c6385ca0748a5336b18efff15d7064f3da5c Mon Sep 17 00:00:00 2001
From: Sree Kuchibhotla <sreek@google.com>
Date: Mon, 4 Jan 2016 15:39:27 -0800
Subject: [PATCH 8/8] remove unused code

---
 tools/run_tests/run_stress_tests.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index 0dc38986ee9..b01a07af901 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -228,13 +228,6 @@ argp.add_argument(
 argp.add_argument('--test_duration_secs',
                   help='The duration of the test in seconds',
                   default=_DEFAULT_TEST_DURATION_SECS)
-argp.add_argument(
-    '--allow_flakes',
-    default=False,
-    action='store_const',
-    const=True,
-    help=
-    'Allow flaky tests to show as passing (re-runs failed tests up to five times)')
 
 args = argp.parse_args()