Merge branch 'master' into async_thread_stress_test

Makefile

@@ -208,7 +208,7 @@ CFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-fr
 CXXFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie
 LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,)
 DEFINES_msan = NDEBUG
-DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5
+DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=2
 
 VALID_CONFIG_mutrace = 1
 CC_mutrace = $(DEFAULT_CC)

build.yaml

@@ -2561,7 +2561,7 @@ configs:
       -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,)
     LDXX: clang++
     compile_the_world: true
-    timeout_multiplier: 1.5
+    timeout_multiplier: 2
   mutrace:
     CPPFLAGS: -O0
     DEFINES: _DEBUG DEBUG

setup.py

@@ -119,6 +119,7 @@ PACKAGE_DIRECTORIES = {
 }
 
 INSTALL_REQUIRES = (
+    'six>=1.10',
     'enum34>=1.0.4',
     'futures>=2.2.0',
     # TODO(atash): eventually split the grpcio package into a metapackage
@@ -131,6 +132,7 @@ SETUP_REQUIRES = (
 ) + INSTALL_REQUIRES
 
 COMMAND_CLASS = {
+    'install': commands.Install,
     'doc': commands.SphinxDocumentation,
     'build_proto_modules': commands.BuildProtoModules,
     'build_project_metadata': commands.BuildProjectMetadata,
@@ -138,6 +140,7 @@ COMMAND_CLASS = {
     'build_ext': commands.BuildExt,
     'gather': commands.Gather,
     'run_interop': commands.RunInterop,
+    'bdist_egg_grpc_custom': commands.BdistEggCustomName,
 }
 
 # Ensure that package data is copied over before any commands have been run:
@@ -187,7 +190,7 @@ else:
 
 setuptools.setup(
     name='grpcio',
-    version='0.12.0b5',
+    version='0.12.0b6',
     license=LICENSE,
     ext_modules=CYTHON_EXTENSION_MODULES,
     packages=list(PACKAGES),

src/core/client_config/subchannel.c

@@ -284,9 +284,13 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector,
   c->connector = connector;
   grpc_connector_ref(c->connector);
   c->num_filters = args->filter_count;
-  c->filters = gpr_malloc(sizeof(grpc_channel_filter *) * c->num_filters);
-  memcpy((void *)c->filters, args->filters,
-         sizeof(grpc_channel_filter *) * c->num_filters);
+  if (c->num_filters > 0) {
+    c->filters = gpr_malloc(sizeof(grpc_channel_filter *) * c->num_filters);
+    memcpy((void *)c->filters, args->filters,
+           sizeof(grpc_channel_filter *) * c->num_filters);
+  } else {
+    c->filters = NULL;
+  }
   c->addr = gpr_malloc(args->addr_len);
   memcpy(c->addr, args->addr, args->addr_len);
   grpc_pollset_set_init(&c->pollset_set);
@@ -483,7 +487,9 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   /* build final filter list */
   num_filters = c->num_filters + c->connecting_result.num_filters + 1;
   filters = gpr_malloc(sizeof(*filters) * num_filters);
-  memcpy((void *)filters, c->filters, sizeof(*filters) * c->num_filters);
+  if (c->num_filters > 0) {
+    memcpy((void *)filters, c->filters, sizeof(*filters) * c->num_filters);
+  }
   memcpy((void *)(filters + c->num_filters), c->connecting_result.filters,
          sizeof(*filters) * c->connecting_result.num_filters);
   filters[num_filters - 1] = &grpc_connected_channel_filter;

src/python/grpcio/commands.py

@@ -30,15 +30,22 @@
 """Provides distutils command classes for the GRPC Python setup process."""
 
 import distutils
+import glob
 import os
 import os.path
+import platform
 import re
+import shutil
 import subprocess
 import sys
+import traceback
 
 import setuptools
+from setuptools.command import bdist_egg
 from setuptools.command import build_ext
 from setuptools.command import build_py
+from setuptools.command import easy_install
+from setuptools.command import install
 from setuptools.command import test
 
 import support
@@ -58,6 +65,129 @@ class CommandError(Exception):
   """Simple exception class for GRPC custom commands."""
 
 
+# TODO(atash): Remove this once PyPI has better Linux bdist support. See
+# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
+def _get_linux_bdist_egg(decorated_basename, target_egg_basename):
+  """Returns a string path to a .egg file for Linux to install.
+
+  If we can retrieve a pre-compiled egg from online, uses it. Else, emits a
+  warning and builds from source.
+  """
+  # Break import style to ensure that setup.py has had a chance to install the
+  # relevant package eggs.
+  from six.moves.urllib import request
+  decorated_path = decorated_basename + '.egg'
+  try:
+    url = (
+        'https://storage.googleapis.com/grpc-precompiled-binaries/'
+        'python/{target}'
+            .format(target=decorated_path))
+    egg_data = request.urlopen(url).read()
+  except IOError as error:
+    raise CommandError(
+        '{}\n\nCould not find the bdist egg {}: {}'
+            .format(traceback.format_exc(), decorated_path, error.message))
+  # Our chosen local egg path.
+  egg_path = target_egg_basename + '.egg'
+  try:
+    with open(egg_path, 'w') as egg_file:
+      egg_file.write(egg_data)
+  except IOError as error:
+    raise CommandError(
+        '{}\n\nCould not write grpcio egg: {}'
+            .format(traceback.format_exc(), error.message))
+  return egg_path
+
+
+class EggNameMixin(object):
+
+  def egg_name(self, with_custom):
+    """
+    Args:
+      with_custom: Boolean describing whether or not to decorate the egg name
+        with custom gRPC-specific target information.
+    """
+    egg_command = self.get_finalized_command('bdist_egg')
+    base = os.path.splitext(os.path.basename(egg_command.egg_output))[0]
+    if with_custom:
+      flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
+      return '{base}-{flavor}'.format(base=base, flavor=flavor)
+    else:
+      return base
+
+
+class Install(install.install, EggNameMixin):
+  """Custom Install command for gRPC Python.
+
+  This is for bdist shims and whatever else we might need a custom install
+  command for.
+  """
+
+  user_options = install.install.user_options + [
+      # TODO(atash): remove this once manylinux gets on PyPI. See
+      # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
+      ('use-linux-bdist', None,
+       'Whether to retrieve a binary for Linux instead of building from '
+       'source.'),
+  ]
+
+  def initialize_options(self):
+    install.install.initialize_options(self)
+    self.use_linux_bdist = False
+
+  def finalize_options(self):
+    install.install.finalize_options(self)
+
+  def run(self):
+    if self.use_linux_bdist:
+      try:
+        egg_path = _get_linux_bdist_egg(self.egg_name(True),
+                                        self.egg_name(False))
+      except CommandError as error:
+        sys.stderr.write(
+            '\nWARNING: Failed to acquire grpcio prebuilt binary:\n'
+            '{}.\n\n'.format(error.message))
+        raise
+      try:
+        self._run_bdist_retrieval_install(egg_path)
+      except Exception as error:
+        # if anything else happens (and given how there's no way to really know
+        # what's happening in setuptools here, I mean *anything*), warn the user
+        # and fall back to building from source.
+        sys.stderr.write(
+            '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n'
+                .format(traceback.format_exc()))
+        install.install.run(self)
+    else:
+      install.install.run(self)
+
+  # TODO(atash): Remove this once PyPI has better Linux bdist support. See
+  # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
+  def _run_bdist_retrieval_install(self, bdist_egg):
+    easy_install = self.distribution.get_command_class('easy_install')
+    easy_install_command = easy_install(
+        self.distribution, args='x', root=self.root, record=self.record,
+    )
+    easy_install_command.ensure_finalized()
+    easy_install_command.always_copy_from = '.'
+    easy_install_command.package_index.scan(glob.glob('*.egg'))
+    arguments = [bdist_egg]
+    if setuptools.bootstrap_install_from:
+      args.insert(0, setuptools.bootstrap_install_from)
+    easy_install_command.args = arguments
+    easy_install_command.run()
+    setuptools.bootstrap_install_from = None
+
+
+class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin):
+  """Thin wrapper around the bdist_egg command to build with our custom name."""
+
+  def run(self):
+    bdist_egg.bdist_egg.run(self)
+    target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True)))
+    shutil.move(self.get_outputs()[0], target)
+
+
 class SphinxDocumentation(setuptools.Command):
   """Command to generate documentation via sphinx."""
 

test/cpp/qps/client_async.cc

@@ -46,13 +46,14 @@
 #include <grpc++/client_context.h>
 #include <grpc++/generic/generic_stub.h>
 #include <grpc/grpc.h>
+#include <grpc/support/cpu.h>
 #include <grpc/support/histogram.h>
 #include <grpc/support/log.h>
 
+#include "src/proto/grpc/testing/services.grpc.pb.h"
 #include "test/cpp/qps/client.h"
 #include "test/cpp/qps/timer.h"
 #include "test/cpp/util/create_test_channel.h"
-#include "src/proto/grpc/testing/services.grpc.pb.h"
 
 namespace grpc {
 namespace testing {
@@ -164,14 +165,15 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
               std::function<std::unique_ptr<StubType>(std::shared_ptr<Channel>)>
                   create_stub)
       : ClientImpl<StubType, RequestType>(config, create_stub),
+        num_async_threads_(NumThreads(config)),
         channel_lock_(new std::mutex[config.client_channels()]),
         contexts_(config.client_channels()),
         max_outstanding_per_channel_(config.outstanding_rpcs_per_channel()),
         channel_count_(config.client_channels()),
-        pref_channel_inc_(config.async_client_threads()) {
-    SetupLoadTest(config, config.async_client_threads());
+        pref_channel_inc_(num_async_threads_) {
+    SetupLoadTest(config, num_async_threads_);
 
-    for (int i = 0; i < config.async_client_threads(); i++) {
+    for (int i = 0; i < num_async_threads_; i++) {
       cli_cqs_.emplace_back(new CompletionQueue);
       if (!closed_loop_) {
         rpc_deadlines_.emplace_back();
@@ -324,6 +326,9 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
     return true;
   }
 
+ protected:
+  int num_async_threads_;
+
  private:
   class boolean {  // exists only to avoid data-race on vector<bool>
    public:
@@ -338,6 +343,15 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
    private:
     bool val_;
   };
+  static int NumThreads(const ClientConfig& config) {
+    int num_threads = config.async_client_threads();
+    if (num_threads <= 0) {  // Use dynamic sizing
+      num_threads = gpr_cpu_num_cores();
+      gpr_log(GPR_INFO, "Sizing client server to %d threads", num_threads);
+    }
+    return num_threads;
+  }
+
   std::vector<std::unique_ptr<CompletionQueue>> cli_cqs_;
 
   std::vector<deadline_list> rpc_deadlines_;  // per thread deadlines
@@ -363,7 +377,7 @@ class AsyncUnaryClient GRPC_FINAL
  public:
   explicit AsyncUnaryClient(const ClientConfig& config)
       : AsyncClient(config, SetupCtx, BenchmarkStubCreator) {
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
   ~AsyncUnaryClient() GRPC_OVERRIDE { EndThreads(); }
 
@@ -461,7 +475,7 @@ class AsyncStreamingClient GRPC_FINAL
     // async streaming currently only supports closed loop
     GPR_ASSERT(closed_loop_);
 
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
 
   ~AsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); }
@@ -566,7 +580,7 @@ class GenericAsyncStreamingClient GRPC_FINAL
     // async streaming currently only supports closed loop
     GPR_ASSERT(closed_loop_);
 
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
 
   ~GenericAsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); }

test/cpp/qps/driver.cc

@@ -31,24 +31,24 @@
  *
  */
 
+#include <deque>
 #include <list>
 #include <thread>
-#include <deque>
 #include <vector>
 
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/host_port.h>
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
 
 #include "src/core/support/env.h"
+#include "src/proto/grpc/testing/services.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/histogram.h"
 #include "test/cpp/qps/qps_worker.h"
-#include "src/proto/grpc/testing/services.grpc.pb.h"
 
 using std::list;
 using std::thread;
@@ -142,6 +142,12 @@ std::unique_ptr<ScenarioResult> RunScenario(
     }
   }
 
+  // if num_clients is set to <=0, do dynamic sizing: all workers
+  // except for servers are clients
+  if (num_clients <= 0) {
+    num_clients = workers.size() - num_servers;
+  }
+
   // TODO(ctiller): support running multiple configurations, and binpack
   // client/server pairs
   // to available workers

test/cpp/qps/server_async.cc

@@ -50,8 +50,8 @@
 #include <grpc/support/log.h>
 #include <gtest/gtest.h>
 
-#include "test/cpp/qps/server.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
+#include "test/cpp/qps/server.h"
 
 namespace grpc {
 namespace testing {
@@ -85,7 +85,13 @@ class AsyncQpsServerTest : public Server {
 
     register_service(&builder, &async_service_);
 
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    int num_threads = config.async_server_threads();
+    if (num_threads <= 0) {  // dynamic sizing
+      num_threads = cores();
+      gpr_log(GPR_INFO, "Sizing async server to %d threads", num_threads);
+    }
+
+    for (int i = 0; i < num_threads; i++) {
       srv_cqs_.emplace_back(builder.AddCompletionQueue());
     }
 
@@ -96,8 +102,8 @@ class AsyncQpsServerTest : public Server {
     auto process_rpc_bound =
         std::bind(process_rpc, config.payload_config(), _1, _2);
 
-    for (int i = 0; i < 10000 / config.async_server_threads(); i++) {
-      for (int j = 0; j < config.async_server_threads(); j++) {
+    for (int i = 0; i < 10000 / num_threads; i++) {
+      for (int j = 0; j < num_threads; j++) {
         if (request_unary_function) {
           auto request_unary =
               std::bind(request_unary_function, &async_service_, _1, _2, _3,
@@ -115,10 +121,10 @@ class AsyncQpsServerTest : public Server {
       }
     }
 
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    for (int i = 0; i < num_threads; i++) {
       shutdown_state_.emplace_back(new PerThreadShutdownState());
     }
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    for (int i = 0; i < num_threads; i++) {
       threads_.emplace_back(&AsyncQpsServerTest::ThreadFunc, this, i);
     }
   }

tools/dockerfile/grpc_sanity/Dockerfile

@@ -0,0 +1,79 @@
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Dockerfile for running gRPC sanity tests
+
+FROM debian:jessie
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  gcc-multilib \
+  git \
+  golang \
+  gyp \
+  lcov \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  perl \
+  strace \
+  python-dev \
+  python-setuptools \
+  python-yaml \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+##################
+# Sanity test dependencies
+RUN apt-get update && apt-get install -y python-pip
+RUN pip install simplejson mako
+
+##################
+# Docker "inception".
+# Note this is quite the ugly hack.
+# This makes sure that the docker binary we inject has its dependencies.
+RUN curl https://get.docker.com/ | sh
+RUN apt-get remove --purge -y docker-engine
+
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/grpc_tests_multilang_x64/Dockerfile

@@ -27,8 +27,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# A work-in-progress Dockerfile that allows running gRPC test suites
-# inside a docker container.
+# Dockerfile for running gRPC test suites inside a docker container.
 
 FROM debian:jessie
 
@@ -95,8 +94,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \
 RUN cd /var/local && wget www.nuget.org/NuGet.exe
 ENV NUGET mono /var/local/NuGet.exe
 
-# TODO(jtattermusch): add dependencies for other languages
-
 ##################
 # Node dependencies
 
@@ -154,18 +151,8 @@ RUN apt-get update && apt-get install -y \
 
 ##################
 # Zookeeper dependencies
-
-# Install dependencies
-
 RUN apt-get install -y libzookeeper-mt-dev
 
-##################
-# Docker "inception".
-# Note this is quite the ugly hack.
-# This makes sure that the docker binary we inject has its dependencies.
-RUN curl https://get.docker.com/ | sh
-RUN apt-get remove --purge -y docker-engine
-
 RUN mkdir /var/local/jenkins
 
 # Define the default command.

tools/dockerfile/grpc_tests_multilang_x86/Dockerfile

@@ -27,8 +27,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# A work-in-progress Dockerfile that allows running gRPC test suites
-# inside a docker container.
+# Dockerfile for running gRPC test suites inside a docker container.
 
 FROM 32bit/debian:jessie
 
@@ -95,8 +94,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \
 RUN cd /var/local && wget www.nuget.org/NuGet.exe
 ENV NUGET mono /var/local/NuGet.exe
 
-# TODO(jtattermusch): add dependencies for other languages
-
 ##################
 # Node dependencies
 
@@ -153,9 +150,6 @@ RUN apt-get update && apt-get install -y \
 
 ##################
 # Zookeeper dependencies
-
-# Install dependencies
-
 RUN apt-get install -y libzookeeper-mt-dev
 
 

tools/jenkins/docker_run_tests.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -43,8 +43,12 @@ chown `whoami` $XDG_CACHE_HOME
 mkdir -p /var/local/git
 git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
 
-nvm use 0.12
-rvm use ruby-2.1
+nvm use 0.12 || true
+
+if [ -x "$(command -v rvm)" ]
+then
+  rvm use ruby-2.1
+fi
 
 mkdir -p reports
 

tools/jenkins/run_jenkins.sh

@@ -49,7 +49,7 @@ fi
 
 unset platform  # variable named 'platform' breaks the windows build
 
-python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true"
+python tools/run_tests/run_tests.py $USE_DOCKER_MAYBE -t -l $language -c $config -x report.xml -j 2 $@ || TESTS_FAILED="true"
 
 if [ ! -e reports/index.html ]
 then

tools/run_tests/configs.json

@@ -59,7 +59,7 @@
   }, 
   {
     "config": "msan", 
-    "timeout_multiplier": 1.5
+    "timeout_multiplier": 2
   }, 
   {
     "config": "mutrace"

tools/run_tests/jobset.py

@@ -360,7 +360,7 @@ class Jobset(object):
       if self.cancelled(): return False
       current_cpu_cost = self.cpu_cost()
       if current_cpu_cost == 0: break
-      if current_cpu_cost + spec.cpu_cost < self._maxjobs: break
+      if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break
       self.reap()
     if self.cancelled(): return False
     if spec.hash_targets:

tools/run_tests/run_tests.py

@@ -183,6 +183,9 @@ class CLanguage(object):
   def supports_multi_config(self):
     return True
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return self.make_target
 
@@ -215,6 +218,9 @@ class NodeLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'node'
 
@@ -246,6 +252,9 @@ class PhpLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'php'
 
@@ -299,6 +308,9 @@ class PythonLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'python'
 
@@ -330,6 +342,9 @@ class RubyLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'ruby'
 
@@ -412,6 +427,9 @@ class CSharpLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'csharp'
 
@@ -443,6 +461,9 @@ class ObjCLanguage(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return 'objc'
 
@@ -451,8 +472,10 @@ class Sanity(object):
 
   def test_specs(self, config, args):
     import yaml
-    with open('tools/run_tests/sanity_tests.yaml', 'r') as f:
-      return [config.job_spec(cmd['script'].split(), None, timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1))
+    with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
+      return [config.job_spec(cmd['script'].split(), None,
+                              timeout_seconds=None, environ={'TEST': 'true'},
+                              cpu_cost=cmd.get('cpu_cost', 1))
               for cmd in yaml.load(f)]
 
   def pre_build_steps(self):
@@ -476,6 +499,9 @@ class Sanity(object):
   def supports_multi_config(self):
     return False
 
+  def dockerfile_dir(self, config, arch):
+    return 'tools/dockerfile/grpc_sanity'
+
   def __str__(self):
     return 'sanity'
 
@@ -506,6 +532,9 @@ class Build(object):
   def supports_multi_config(self):
     return True
 
+  def dockerfile_dir(self, config, arch):
+    return None
+
   def __str__(self):
     return self.make_target
 
@@ -545,7 +574,7 @@ def _windows_arch_option(arch):
   else:
     print 'Architecture %s not supported.' % arch
     sys.exit(1)
-    
+
 
 def _check_arch_option(arch):
   """Checks that architecture option is valid."""
@@ -595,15 +624,19 @@ def _windows_toolset_option(compiler):
     sys.exit(1)
 
 
-def _get_dockerfile_dir(arch):
+def _get_dockerfile_dir(language, cfg, arch):
   """Returns dockerfile to use"""
-  if arch == 'default' or arch == 'x64':
-    return 'tools/dockerfile/grpc_jenkins_slave_x64'
-  elif arch == 'x86':
-    return 'tools/dockerfile/grpc_jenkins_slave_x86'
+  custom = language.dockerfile_dir(cfg, arch)
+  if custom:
+    return custom
   else:
-    print 'Architecture %s not supported with current settings.' % arch
-    sys.exit(1)
+    if arch == 'default' or arch == 'x64':
+      return 'tools/dockerfile/grpc_tests_multilang_x64'
+    elif arch == 'x86':
+      return 'tools/dockerfile/grpc_tests_multilang_x86'
+    else:
+      print 'Architecture %s not supported with current settings.' % arch
+      sys.exit(1)
 
 def runs_per_test_type(arg_str):
     """Auxilary function to parse the "runs_per_test" flag.
@@ -771,11 +804,13 @@ if args.use_docker:
     time.sleep(5)
 
   child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ]
-  run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])
+  run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:])
 
   env = os.environ.copy()
   env['RUN_TESTS_COMMAND'] = run_tests_cmd
-  env['DOCKERFILE_DIR'] = _get_dockerfile_dir(args.arch)
+  env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)),
+                                              next(iter(build_configs)),
+                                              args.arch)
   env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh'
   if args.xml_report:
     env['XML_REPORT'] = args.xml_report
@@ -786,7 +821,7 @@ if args.use_docker:
                         shell=True,
                         env=env)
   sys.exit(0)
-  
+
 if platform_string() != 'windows' and args.compiler != 'default':
     print 'Compiler %s not supported on current platform.' % args.compiler
     sys.exit(1)

tools/run_tests/sanity/check_sources_and_headers.py

@@ -33,9 +33,9 @@ import os
 import re
 import sys
 
-root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
 with open(os.path.join(root, 'tools', 'run_tests', 'sources_and_headers.json')) as f:
-	js = json.loads(f.read())
+  js = json.loads(f.read())
 
 re_inc1 = re.compile(r'^#\s*include\s*"([^"]*)"')
 assert re_inc1.match('#include "foo"').group(1) == 'foo'
@@ -43,41 +43,41 @@ re_inc2 = re.compile(r'^#\s*include\s*<((grpc|grpc\+\+)/[^"]*)>')
 assert re_inc2.match('#include <grpc++/foo>').group(1) == 'grpc++/foo'
 
 def get_target(name):
-	for target in js:
-		if target['name'] == name:
-			return target
-	assert False, 'no target %s' % name
+  for target in js:
+    if target['name'] == name:
+      return target
+  assert False, 'no target %s' % name
 
 def target_has_header(target, name):
-#	print target['name'], name
-	if name in target['headers']:
-		return True
-	for dep in target['deps']:
-		if target_has_header(get_target(dep), name):
-			return True
-	if name == 'src/core/profiling/stap_probes.h':
-		return True
-	return False
+  # print target['name'], name
+  if name in target['headers']:
+    return True
+  for dep in target['deps']:
+    if target_has_header(get_target(dep), name):
+      return True
+  if name == 'src/core/profiling/stap_probes.h':
+    return True
+  return False
 
 errors = 0
 for target in js:
-	for fn in target['src']:
-		with open(os.path.join(root, fn)) as f:
-			src = f.read().splitlines()
-		for line in src:
-			m = re_inc1.match(line)
-			if m:
-				if not target_has_header(target, m.group(1)):
-					print (
-						'target %s (%s) does not name header %s as a dependency' % (
-							target['name'], fn, m.group(1)))
-					errors += 1
-			m = re_inc2.match(line)
-			if m:
-				if not target_has_header(target, 'include/' + m.group(1)):
-					print (
-						'target %s (%s) does not name header %s as a dependency' % (
-							target['name'], fn, m.group(1)))
-					errors += 1
+  for fn in target['src']:
+    with open(os.path.join(root, fn)) as f:
+      src = f.read().splitlines()
+    for line in src:
+      m = re_inc1.match(line)
+      if m:
+        if not target_has_header(target, m.group(1)):
+          print (
+            'target %s (%s) does not name header %s as a dependency' % (
+              target['name'], fn, m.group(1)))
+          errors += 1
+      m = re_inc2.match(line)
+      if m:
+        if not target_has_header(target, 'include/' + m.group(1)):
+          print (
+            'target %s (%s) does not name header %s as a dependency' % (
+              target['name'], fn, m.group(1)))
+          errors += 1
 
 assert errors == 0

tools/run_tests/sanity/check_submodules.sh

@@ -34,7 +34,7 @@ set -e
 
 export TEST=true
 
-cd `dirname $0`/../..
+cd `dirname $0`/../../..
 
 submodules=`mktemp /tmp/submXXXXXX`
 want_submodules=`mktemp /tmp/submXXXXXX`

tools/run_tests/sanity/sanity_tests.yaml

@@ -1,7 +1,7 @@
 # a set of tests that are run in parallel for sanity tests
-- script: tools/run_tests/check_sources_and_headers.py
-- script: tools/run_tests/check_submodules.sh
-- script: tools/run_tests/check_cache_mk.sh
+- script: tools/run_tests/sanity/check_sources_and_headers.py
+- script: tools/run_tests/sanity/check_submodules.sh
+- script: tools/run_tests/sanity/check_cache_mk.sh
 - script: tools/buildgen/generate_projects.sh -j 3
   cpu_cost: 3
 - script: tools/distrib/check_copyright.py