From 930c2983d34507765305a56c329c2e354168184b Mon Sep 17 00:00:00 2001
From: Dan Born <dborn@google.com>
Date: Thu, 8 Sep 2016 19:12:18 -0700
Subject: [PATCH 01/52] Safe server shutdown.

---
 .../server/secure/server_secure_chttp2.c      | 196 ++++++++++--------
 src/core/lib/iomgr/tcp_server_posix.c         |  36 ++--
 test/core/iomgr/tcp_server_posix_test.c       |   3 +-
 3 files changed, 131 insertions(+), 104 deletions(-)

diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index da3e284fcf2..563271f4f8c 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -61,13 +61,12 @@ typedef struct server_secure_state {
   grpc_server_credentials *creds;
   bool is_shutdown;
   gpr_mu mu;
-  gpr_refcount refcount;
-  grpc_closure destroy_closure;
-  grpc_closure *destroy_callback;
+  grpc_closure tcp_server_shutdown_complete;
+  grpc_closure *server_destroy_listener_done;
 } server_secure_state;
 
 typedef struct server_secure_connect {
-  server_secure_state *state;
+  server_secure_state *server_state;
   grpc_pollset *accepting_pollset;
   grpc_tcp_server_acceptor *acceptor;
   grpc_handshake_manager *handshake_mgr;
@@ -77,39 +76,28 @@ typedef struct server_secure_connect {
   grpc_channel_args *args;
 } server_secure_connect;
 
-static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); }
-
-static void state_unref(server_secure_state *state) {
-  if (gpr_unref(&state->refcount)) {
-    /* ensure all threads have unlocked */
-    gpr_mu_lock(&state->mu);
-    gpr_mu_unlock(&state->mu);
-    /* clean up */
-    GRPC_SECURITY_CONNECTOR_UNREF(&state->sc->base, "server");
-    grpc_server_credentials_unref(state->creds);
-    gpr_free(state);
-  }
-}
-
 static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
                                      grpc_security_status status,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
-  server_secure_connect *state = statep;
+  server_secure_connect *connection_state = statep;
   if (status == GRPC_SECURITY_OK) {
     if (secure_endpoint) {
-      gpr_mu_lock(&state->state->mu);
-      if (!state->state->is_shutdown) {
+      gpr_mu_lock(&connection_state->server_state->mu);
+      if (!connection_state->server_state->is_shutdown) {
         grpc_transport *transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(state->state->server),
+            exec_ctx, grpc_server_get_channel_args(
+                          connection_state->server_state->server),
             secure_endpoint, 0);
         grpc_arg args_to_add[2];
-        args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds);
+        args_to_add[0] = grpc_server_credentials_to_arg(
+            connection_state->server_state->creds);
         args_to_add[1] = grpc_auth_context_to_arg(auth_context);
         grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
-            state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add));
-        grpc_server_setup_transport(exec_ctx, state->state->server, transport,
-                                    state->accepting_pollset, args_copy);
+            connection_state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add));
+        grpc_server_setup_transport(
+            exec_ctx, connection_state->server_state->server, transport,
+            connection_state->accepting_pollset, args_copy);
         grpc_channel_args_destroy(args_copy);
         grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL);
       } else {
@@ -117,21 +105,21 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
          * gone away. */
         grpc_endpoint_destroy(exec_ctx, secure_endpoint);
       }
-      gpr_mu_unlock(&state->state->mu);
+      gpr_mu_unlock(&connection_state->server_state->mu);
     }
   } else {
     gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
   }
-  grpc_channel_args_destroy(state->args);
-  state_unref(state->state);
-  gpr_free(state);
+  grpc_channel_args_destroy(connection_state->args);
+  grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
+  gpr_free(connection_state);
 }
 
 static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
                               grpc_channel_args *args,
                               gpr_slice_buffer *read_buffer, void *user_data,
                               grpc_error *error) {
-  server_secure_connect *state = user_data;
+  server_secure_connect *connection_state = user_data;
   if (error != GRPC_ERROR_NONE) {
     const char *error_str = grpc_error_string(error);
     gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
@@ -139,81 +127,107 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
     GRPC_ERROR_UNREF(error);
     grpc_channel_args_destroy(args);
     gpr_free(read_buffer);
-    grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr);
-    grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
-    state_unref(state->state);
-    gpr_free(state);
+    grpc_handshake_manager_shutdown(exec_ctx, connection_state->handshake_mgr);
+    grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
+    grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
+    gpr_free(connection_state);
     return;
   }
-  grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
-  state->handshake_mgr = NULL;
+  grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
+  connection_state->handshake_mgr = NULL;
   // TODO(roth, jboeuf): Convert security connector handshaking to use new
   // handshake API, and then move the code from on_secure_handshake_done()
   // into this function.
-  state->args = args;
+  connection_state->args = args;
   grpc_server_security_connector_do_handshake(
-      exec_ctx, state->state->sc, state->acceptor, endpoint, read_buffer,
-      state->deadline, on_secure_handshake_done, state);
+      exec_ctx, connection_state->server_state->sc, connection_state->acceptor,
+      endpoint, read_buffer, connection_state->deadline,
+      on_secure_handshake_done, connection_state);
 }
 
 static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
                       grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
-  server_secure_connect *state = gpr_malloc(sizeof(*state));
-  state->state = statep;
-  state_ref(state->state);
-  state->accepting_pollset = accepting_pollset;
-  state->acceptor = acceptor;
-  state->handshake_mgr = grpc_handshake_manager_create();
+  server_secure_state *server_state = statep;
+  server_secure_connect *connection_state = NULL;
+  gpr_mu_lock(&server_state->mu);
+  if (server_state->is_shutdown) {
+    gpr_mu_unlock(&server_state->mu);
+    grpc_endpoint_destroy(exec_ctx, tcp);
+    return;
+  }
+  gpr_mu_unlock(&server_state->mu);
+  grpc_tcp_server_ref(server_state->tcp);
+  connection_state = gpr_malloc(sizeof(*connection_state));
+  connection_state->server_state = server_state;
+  connection_state->accepting_pollset = accepting_pollset;
+  connection_state->acceptor = acceptor;
+  connection_state->handshake_mgr = grpc_handshake_manager_create();
   // TODO(roth): We should really get this timeout value from channel
   // args instead of hard-coding it.
-  state->deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
-                                 gpr_time_from_seconds(120, GPR_TIMESPAN));
+  connection_state->deadline = gpr_time_add(
+      gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
   grpc_handshake_manager_do_handshake(
-      exec_ctx, state->handshake_mgr, tcp,
-      grpc_server_get_channel_args(state->state->server), state->deadline,
-      acceptor, on_handshake_done, state);
+      exec_ctx, connection_state->handshake_mgr, tcp,
+      grpc_server_get_channel_args(connection_state->server_state->server),
+      connection_state->deadline, acceptor, on_handshake_done,
+      connection_state);
 }
 
 /* Server callback: start listening on our ports */
-static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
-                  grpc_pollset **pollsets, size_t pollset_count) {
-  server_secure_state *state = statep;
-  grpc_tcp_server_start(exec_ctx, state->tcp, pollsets, pollset_count,
-                        on_accept, state);
+static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server,
+                                  void *statep, grpc_pollset **pollsets,
+                                  size_t pollset_count) {
+  server_secure_state *server_state = statep;
+  gpr_mu_lock(&server_state->mu);
+  server_state->is_shutdown = false;
+  gpr_mu_unlock(&server_state->mu);
+  grpc_tcp_server_start(exec_ctx, server_state->tcp, pollsets, pollset_count,
+                        on_accept, server_state);
 }
 
-static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep,
-                         grpc_error *error) {
-  server_secure_state *state = statep;
-  if (state->destroy_callback != NULL) {
-    state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg,
-                                GRPC_ERROR_REF(error));
+static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *statep,
+                                         grpc_error *error) {
+  server_secure_state *server_state = statep;
+  /* ensure all threads have unlocked */
+  gpr_mu_lock(&server_state->mu);
+  grpc_closure *destroy_done = server_state->server_destroy_listener_done;
+  GPR_ASSERT(server_state->is_shutdown);
+  gpr_mu_unlock(&server_state->mu);
+  /* clean up */
+  grpc_server_security_connector_shutdown(exec_ctx, server_state->sc);
+
+  /* Flush queued work before a synchronous unref. */
+  grpc_exec_ctx_flush(exec_ctx);
+  GRPC_SECURITY_CONNECTOR_UNREF(&server_state->sc->base, "server");
+  grpc_server_credentials_unref(server_state->creds);
+
+  if (destroy_done != NULL) {
+    destroy_done->cb(exec_ctx, destroy_done->cb_arg, GRPC_ERROR_REF(error));
+    grpc_exec_ctx_flush(exec_ctx);
   }
-  grpc_server_security_connector_shutdown(exec_ctx, state->sc);
-  state_unref(state);
+  gpr_free(server_state);
 }
 
-/* Server callback: destroy the tcp listener (so we don't generate further
-   callbacks) */
-static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
-                    grpc_closure *callback) {
-  server_secure_state *state = statep;
+static void server_destroy_listener(grpc_exec_ctx *exec_ctx,
+                                    grpc_server *server, void *statep,
+                                    grpc_closure *callback) {
+  server_secure_state *server_state = statep;
   grpc_tcp_server *tcp;
-  gpr_mu_lock(&state->mu);
-  state->is_shutdown = true;
-  state->destroy_callback = callback;
-  tcp = state->tcp;
-  gpr_mu_unlock(&state->mu);
+  gpr_mu_lock(&server_state->mu);
+  server_state->is_shutdown = true;
+  server_state->server_destroy_listener_done = callback;
+  tcp = server_state->tcp;
+  gpr_mu_unlock(&server_state->mu);
   grpc_tcp_server_shutdown_listeners(exec_ctx, tcp);
-  grpc_tcp_server_unref(exec_ctx, tcp);
+  grpc_tcp_server_unref(exec_ctx, server_state->tcp);
 }
 
 int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
                                       grpc_server_credentials *creds) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
-  server_secure_state *state = NULL;
+  server_secure_state *server_state = NULL;
   size_t i;
   size_t count = 0;
   int port_num = -1;
@@ -253,22 +267,22 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   if (err != GRPC_ERROR_NONE) {
     goto error;
   }
-  state = gpr_malloc(sizeof(*state));
-  memset(state, 0, sizeof(*state));
-  grpc_closure_init(&state->destroy_closure, destroy_done, state);
-  err = grpc_tcp_server_create(&state->destroy_closure,
+  server_state = gpr_malloc(sizeof(*server_state));
+  memset(server_state, 0, sizeof(*server_state));
+  grpc_closure_init(&server_state->tcp_server_shutdown_complete,
+                    tcp_server_shutdown_complete, server_state);
+  err = grpc_tcp_server_create(&server_state->tcp_server_shutdown_complete,
                                grpc_server_get_channel_args(server), &tcp);
   if (err != GRPC_ERROR_NONE) {
     goto error;
   }
 
-  state->server = server;
-  state->tcp = tcp;
-  state->sc = sc;
-  state->creds = grpc_server_credentials_ref(creds);
-  state->is_shutdown = false;
-  gpr_mu_init(&state->mu);
-  gpr_ref_init(&state->refcount, 1);
+  server_state->server = server;
+  server_state->tcp = tcp;
+  server_state->sc = sc;
+  server_state->creds = grpc_server_credentials_ref(creds);
+  server_state->is_shutdown = true;
+  gpr_mu_init(&server_state->mu);
 
   errors = gpr_malloc(sizeof(*errors) * resolved->naddrs);
   for (i = 0; i < resolved->naddrs; i++) {
@@ -313,7 +327,8 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   grpc_resolved_addresses_destroy(resolved);
 
   /* Register with the server only upon success */
-  grpc_server_add_listener(&exec_ctx, server, state, start, destroy);
+  grpc_server_add_listener(&exec_ctx, server, server_state,
+                           server_start_listener, server_destroy_listener);
 
   grpc_exec_ctx_finish(&exec_ctx);
   return port_num;
@@ -334,10 +349,11 @@ error:
     grpc_tcp_server_unref(&exec_ctx, tcp);
   } else {
     if (sc) {
+      grpc_exec_ctx_flush(&exec_ctx);
       GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server");
     }
-    if (state) {
-      gpr_free(state);
+    if (server_state) {
+      gpr_free(server_state);
     }
   }
   grpc_exec_ctx_finish(&exec_ctx);
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index 2d3f6cf9a7d..5f846c8afb1 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -191,6 +191,9 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
+  gpr_mu_lock(&s->mu);
+  GPR_ASSERT(s->shutdown);
+  gpr_mu_unlock(&s->mu);
   if (s->shutdown_complete != NULL) {
     grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
@@ -652,6 +655,7 @@ unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s,
                                        unsigned port_index) {
   unsigned num_fds = 0;
   grpc_tcp_listener *sp;
+  gpr_mu_lock(&s->mu);
   for (sp = s->head; sp && port_index != 0; sp = sp->next) {
     if (!sp->is_sibling) {
       --port_index;
@@ -659,12 +663,15 @@ unsigned grpc_tcp_server_port_fd_count(grpc_tcp_server *s,
   }
   for (; sp; sp = sp->sibling, ++num_fds)
     ;
+  gpr_mu_unlock(&s->mu);
   return num_fds;
 }
 
 int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index,
                             unsigned fd_index) {
   grpc_tcp_listener *sp;
+  int fd;
+  gpr_mu_lock(&s->mu);
   for (sp = s->head; sp && port_index != 0; sp = sp->next) {
     if (!sp->is_sibling) {
       --port_index;
@@ -673,10 +680,12 @@ int grpc_tcp_server_port_fd(grpc_tcp_server *s, unsigned port_index,
   for (; sp && fd_index != 0; sp = sp->sibling, --fd_index)
     ;
   if (sp) {
-    return sp->fd;
+    fd = sp->fd;
   } else {
-    return -1;
+    fd = -1;
   }
+  gpr_mu_unlock(&s->mu);
+  return fd;
 }
 
 void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
@@ -722,7 +731,7 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
 }
 
 grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
-  gpr_ref(&s->refs);
+  gpr_ref_non_zero(&s->refs);
   return s;
 }
 
@@ -736,18 +745,21 @@ void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (gpr_unref(&s->refs)) {
-    /* Complete shutdown_starting work before destroying. */
     grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
+    bool finish_ctx = false;
+    /* FIXME: API allows a NULL exec_ctx, although this might cause us to delete
+       ourself before some enqueued work in some other exec_ctx runs. */
+    if (exec_ctx == NULL) {
+      exec_ctx = &local_exec_ctx;
+      finish_ctx = true;
+    }
+    grpc_tcp_server_shutdown_listeners(exec_ctx, s);
     gpr_mu_lock(&s->mu);
-    grpc_exec_ctx_enqueue_list(&local_exec_ctx, &s->shutdown_starting, NULL);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &s->shutdown_starting, NULL);
     gpr_mu_unlock(&s->mu);
-    if (exec_ctx == NULL) {
-      grpc_exec_ctx_flush(&local_exec_ctx);
-      tcp_server_destroy(&local_exec_ctx, s);
-      grpc_exec_ctx_finish(&local_exec_ctx);
-    } else {
-      grpc_exec_ctx_finish(&local_exec_ctx);
-      tcp_server_destroy(exec_ctx, s);
+    tcp_server_destroy(exec_ctx, s);
+    if (finish_ctx) {
+      grpc_exec_ctx_finish(exec_ctx);
     }
   }
 }
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 6e2d1d0fc9e..6b1dd428a13 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -314,11 +314,10 @@ static void test_connect(unsigned n) {
   GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 0) >= 0);
 
   grpc_tcp_server_unref(&exec_ctx, s);
+  grpc_exec_ctx_finish(&exec_ctx);
 
   /* Weak ref lost. */
   GPR_ASSERT(weak_ref.server == NULL);
-
-  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p,

From 1ed0b8e3d72abcc4788e89cab4caa4e8c0083985 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 14 Sep 2016 15:01:16 -0700
Subject: [PATCH 02/52] Add interop test for Cacheable Unary Calls

modified interop test spec doc
added CacheableUnaryCall to test.proto
modified server and client implmenentations to support new method
---
 doc/interop-test-descriptions.md   | 21 +++++++++++++++
 src/proto/grpc/testing/test.proto  |  5 ++++
 test/cpp/interop/client.cc         |  4 +++
 test/cpp/interop/interop_client.cc | 43 ++++++++++++++++++++++++++++++
 test/cpp/interop/interop_client.h  |  1 +
 test/cpp/interop/interop_server.cc | 11 ++++++++
 6 files changed, 85 insertions(+)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 1e04966380c..5b3ad2335cc 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -60,6 +60,27 @@ Client asserts:
 *It may be possible to use UnaryCall instead of EmptyCall, but it is harder to
 ensure that the proto serialized to zero bytes.*
 
+### cacheable_unary
+
+This test verifies that gRPC requests marked as cacheable use GET verb instead
+of POST, and that server sets appropriate cache control headers for the response
+to be cached by a proxy. This interop test requires that the server is behind
+a caching proxy. It is NOT expected to pass if client is accessing the server
+directly.
+
+Server features:
+* [CacheableUnaryCall][]
+
+Procedure:
+ 1. Client calls CacheableUnaryCall twice, and compares the two responses.
+ The server generates a unique response (timestamp) for each request.
+ If the second response was delivered from cache, then both responses will
+ be the same.
+
+Client asserts:
+* call was successful
+* responses are the same.
+
 ### large_unary
 
 This test verifies unary calls succeed in sending messages, and touches on flow
diff --git a/src/proto/grpc/testing/test.proto b/src/proto/grpc/testing/test.proto
index 84369db4b8a..b52c4cbad6c 100644
--- a/src/proto/grpc/testing/test.proto
+++ b/src/proto/grpc/testing/test.proto
@@ -47,6 +47,11 @@ service TestService {
   // One request followed by one response.
   rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
 
+  // One request followed by one response. Response has cache control
+  // headers set such that a caching HTTP proxy (such as GFE) can
+  // satisfy subsequent requests.
+  rpc CacheableUnaryCall(SimpleRequest) returns (SimpleResponse);
+
   // One request followed by a sequence of responses (streamed download).
   // The server returns the payload with client desired type and sizes.
   rpc StreamingOutputCall(StreamingOutputCallRequest)
diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc
index e8ae6ee5723..8cbb1feeafc 100644
--- a/test/cpp/interop/client.cc
+++ b/test/cpp/interop/client.cc
@@ -148,6 +148,8 @@ int main(int argc, char** argv) {
     client.DoStatusWithMessage();
   } else if (FLAGS_test_case == "custom_metadata") {
     client.DoCustomMetadata();
+  } else if (FLAGS_test_case == "cacheable_unary") {
+    client.DoCacheableUnary();
   } else if (FLAGS_test_case == "all") {
     client.DoEmpty();
     client.DoLargeUnary();
@@ -165,6 +167,7 @@ int main(int argc, char** argv) {
     client.DoEmptyStream();
     client.DoStatusWithMessage();
     client.DoCustomMetadata();
+    client.DoCacheableUnary();
     // service_account_creds and jwt_token_creds can only run with ssl.
     if (FLAGS_use_tls) {
       grpc::string json_key = GetServiceAccountJsonKey();
@@ -176,6 +179,7 @@ int main(int argc, char** argv) {
     // compute_engine_creds only runs in GCE.
   } else {
     const char* testcases[] = {"all",
+                               "cacheable_unary",
                                "cancel_after_begin",
                                "cancel_after_first_response",
                                "client_compressed_streaming",
diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index 8861bc1163c..f2290adfc3b 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -845,6 +845,49 @@ bool InteropClient::DoStatusWithMessage() {
   return true;
 }
 
+bool InteropClient::DoCacheableUnary() {
+  gpr_log(GPR_DEBUG, "Sending RPC with cacheable response");
+
+  SimpleRequest request;
+  request.set_response_size(16);
+  grpc::string payload(16, '\0');
+  request.mutable_payload()->set_body(payload.c_str(), 16);
+
+  // Request 1
+  ClientContext context1;
+  SimpleResponse response1;
+  context1.set_cacheable(true);
+  // Add fake user IP since some proxy's (GFE) won't cache requests from
+  // localhost.
+  context1.AddMetadata("x-user-ip", "1.2.3.4");
+  Status s1 =
+      serviceStub_.Get()->CacheableUnaryCall(&context1, request, &response1);
+  if (!AssertStatusOk(s1)) {
+    return false;
+  }
+  gpr_log(GPR_DEBUG, "response 1 payload: %s",
+          response1.payload().body().c_str());
+
+  // Request 2
+  ClientContext context2;
+  SimpleResponse response2;
+  context2.set_cacheable(true);
+  context2.AddMetadata("x-user-ip", "1.2.3.4");
+  Status s2 =
+      serviceStub_.Get()->CacheableUnaryCall(&context2, request, &response2);
+  if (!AssertStatusOk(s2)) {
+    return false;
+  }
+  gpr_log(GPR_DEBUG, "response 1 payload: %s",
+          response2.payload().body().c_str());
+
+  // Check that the body is same for both requests. It will be the same if the
+  // second response is a cached copy of the first response
+  GPR_ASSERT(response2.payload().body() == response1.payload().body());
+
+  return true;
+}
+
 bool InteropClient::DoCustomMetadata() {
   const grpc::string kEchoInitialMetadataKey("x-grpc-test-echo-initial");
   const grpc::string kInitialMetadataValue("test_initial_metadata_value");
diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h
index eb886fcb7e2..1e89f0987d5 100644
--- a/test/cpp/interop/interop_client.h
+++ b/test/cpp/interop/interop_client.h
@@ -79,6 +79,7 @@ class InteropClient {
   bool DoEmptyStream();
   bool DoStatusWithMessage();
   bool DoCustomMetadata();
+  bool DoCacheableUnary();
   // Auth tests.
   // username is a string containing the user email
   bool DoJwtTokenCreds(const grpc::string& username);
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index e5878bb248c..ac2567eba02 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -47,6 +47,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/byte_stream.h"
 #include "src/proto/grpc/testing/empty.grpc.pb.h"
 #include "src/proto/grpc/testing/messages.grpc.pb.h"
@@ -152,6 +153,16 @@ class TestServiceImpl : public TestService::Service {
     return Status::OK;
   }
 
+  // Response contains current timestamp. We ignore everything in the request.
+  Status CacheableUnaryCall(ServerContext* context, const SimpleRequest* request,
+                   SimpleResponse* response) {
+    gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
+    std::string timestamp = std::to_string(ts.tv_nsec);
+    response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
+    context->AddInitialMetadata("Cache-Control", "max-age=100000, public");
+    return Status::OK;
+  }
+
   Status UnaryCall(ServerContext* context, const SimpleRequest* request,
                    SimpleResponse* response) {
     MaybeEchoMetadata(context);

From e26ab6c5610b9cc4878f6db77700b918272dfc80 Mon Sep 17 00:00:00 2001
From: Julien Boeuf <jboeuf@google.com>
Date: Thu, 22 Sep 2016 15:13:07 -0700
Subject: [PATCH 03/52] Adding a method in channel creds to remove any attached
 call creds.

This will be useful when talking to non-trusted load balancer (balancers
which are not able to impersonate real backends) as these balancers
should not receive bearer tokens.
---
 .../composite/composite_credentials.c         | 11 ++++++-
 .../composite/composite_credentials.h         |  4 +--
 .../lib/security/credentials/credentials.c    | 12 +++++++
 .../lib/security/credentials/credentials.h    | 10 ++++++
 .../credentials/fake/fake_credentials.c       |  2 +-
 .../credentials/ssl/ssl_credentials.c         |  2 +-
 test/core/security/credentials_test.c         | 31 +++++++++++++++++--
 7 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 850e41e6460..d55d00b7b66 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -242,8 +242,17 @@ static grpc_security_status composite_channel_create_security_connector(
   return status;
 }
 
+static grpc_channel_credentials *
+composite_channel_duplicate_without_call_credentials(
+    grpc_channel_credentials *creds) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  return grpc_channel_credentials_ref(c->inner_creds);
+}
+
 static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
-    composite_channel_destruct, composite_channel_create_security_connector};
+    composite_channel_destruct, composite_channel_create_security_connector,
+    composite_channel_duplicate_without_call_credentials};
 
 grpc_channel_credentials *grpc_composite_channel_credentials_create(
     grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
index 0d8966f464d..f8425c2b765 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.h
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -53,7 +53,7 @@ grpc_call_credentials *grpc_credentials_contains_type(
     grpc_call_credentials *creds, const char *type,
     grpc_call_credentials **composite_creds);
 
-/* -- Channel composite credentials. -- */
+/* -- Composite channel credentials. -- */
 
 typedef struct {
   grpc_channel_credentials base;
@@ -61,7 +61,7 @@ typedef struct {
   grpc_call_credentials *call_creds;
 } grpc_composite_channel_credentials;
 
-/* -- Composite credentials. -- */
+/* -- Composite call credentials. -- */
 
 typedef struct {
   grpc_call_credentials base;
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
index 029a3572616..1149e5c2edb 100644
--- a/src/core/lib/security/credentials/credentials.c
+++ b/src/core/lib/security/credentials/credentials.c
@@ -138,6 +138,18 @@ grpc_security_status grpc_channel_credentials_create_security_connector(
       channel_creds, NULL, target, args, sc, new_args);
 }
 
+grpc_channel_credentials *
+grpc_channel_credentials_duplicate_without_call_credentials(
+    grpc_channel_credentials *channel_creds) {
+  if (channel_creds != NULL && channel_creds->vtable != NULL &&
+      channel_creds->vtable->duplicate_without_call_credentials != NULL) {
+    return channel_creds->vtable->duplicate_without_call_credentials(
+        channel_creds);
+  } else {
+    return grpc_channel_credentials_ref(channel_creds);
+  }
+}
+
 grpc_server_credentials *grpc_server_credentials_ref(
     grpc_server_credentials *creds) {
   if (creds == NULL) return NULL;
diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h
index 8e9d842eadd..6fb5b5b15ad 100644
--- a/src/core/lib/security/credentials/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -107,6 +107,9 @@ typedef struct {
       grpc_channel_credentials *c, grpc_call_credentials *call_creds,
       const char *target, const grpc_channel_args *args,
       grpc_channel_security_connector **sc, grpc_channel_args **new_args);
+
+  grpc_channel_credentials *(*duplicate_without_call_credentials)(
+      grpc_channel_credentials *c);
 } grpc_channel_credentials_vtable;
 
 struct grpc_channel_credentials {
@@ -128,6 +131,13 @@ grpc_security_status grpc_channel_credentials_create_security_connector(
     const grpc_channel_args *args, grpc_channel_security_connector **sc,
     grpc_channel_args **new_args);
 
+/* Creates a version of the channel credentials without any attached call
+   credentials. This can be used in order to open a channel to a non-trusted
+   gRPC load balancer. */
+grpc_channel_credentials *
+grpc_channel_credentials_duplicate_without_call_credentials(
+    grpc_channel_credentials *creds);
+
 /* --- grpc_credentials_md. --- */
 
 typedef struct {
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
index 51cafd986fb..ea4cb76fb99 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.c
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -61,7 +61,7 @@ fake_transport_security_server_create_security_connector(
 
 static grpc_channel_credentials_vtable
     fake_transport_security_credentials_vtable = {
-        NULL, fake_transport_security_create_security_connector};
+        NULL, fake_transport_security_create_security_connector, NULL};
 
 static grpc_server_credentials_vtable
     fake_transport_security_server_credentials_vtable = {
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c
index 545bca9d98b..0dc1fccec4a 100644
--- a/src/core/lib/security/credentials/ssl/ssl_credentials.c
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c
@@ -95,7 +95,7 @@ static grpc_security_status ssl_create_security_connector(
 }
 
 static grpc_channel_credentials_vtable ssl_vtable = {
-    ssl_destruct, ssl_create_security_connector};
+    ssl_destruct, ssl_create_security_connector, NULL};
 
 static void ssl_build_config(const char *pem_root_certs,
                              grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c
index 7043953154a..2f8ffe4da64 100644
--- a/test/core/security/credentials_test.c
+++ b/test/core/security/credentials_test.c
@@ -46,6 +46,7 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/security/credentials/composite/composite_credentials.h"
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 #include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
 #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
@@ -411,7 +412,7 @@ static grpc_security_status check_channel_oauth2_create_security_connector(
 static void test_channel_oauth2_composite_creds(void) {
   grpc_channel_args *new_args;
   grpc_channel_credentials_vtable vtable = {
-      NULL, check_channel_oauth2_create_security_connector};
+      NULL, check_channel_oauth2_create_security_connector, NULL};
   grpc_channel_credentials *channel_creds =
       grpc_mock_channel_credentials_create(&vtable);
   grpc_call_credentials *oauth2_creds =
@@ -495,7 +496,7 @@ check_channel_oauth2_google_iam_create_security_connector(
 static void test_channel_oauth2_google_iam_composite_creds(void) {
   grpc_channel_args *new_args;
   grpc_channel_credentials_vtable vtable = {
-      NULL, check_channel_oauth2_google_iam_create_security_connector};
+      NULL, check_channel_oauth2_google_iam_create_security_connector, NULL};
   grpc_channel_credentials *channel_creds =
       grpc_mock_channel_credentials_create(&vtable);
   grpc_call_credentials *oauth2_creds =
@@ -1148,6 +1149,31 @@ static void test_get_well_known_google_credentials_file_path(void) {
 #endif
 }
 
+static void test_channel_creds_duplicate_without_call_creds(void) {
+  grpc_channel_credentials *channel_creds =
+      grpc_fake_transport_security_credentials_create();
+
+  grpc_channel_credentials *dup =
+      grpc_channel_credentials_duplicate_without_call_credentials(
+          channel_creds);
+  GPR_ASSERT(dup == channel_creds);
+  grpc_channel_credentials_unref(dup);
+
+  grpc_call_credentials *call_creds =
+      grpc_access_token_credentials_create("blah", NULL);
+  grpc_channel_credentials *composite_creds =
+      grpc_composite_channel_credentials_create(channel_creds, call_creds,
+                                                NULL);
+  grpc_call_credentials_unref(call_creds);
+  dup = grpc_channel_credentials_duplicate_without_call_credentials(
+      composite_creds);
+  GPR_ASSERT(dup == channel_creds);
+  grpc_channel_credentials_unref(dup);
+
+  grpc_channel_credentials_unref(channel_creds);
+  grpc_channel_credentials_unref(composite_creds);
+}
+
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   grpc_init();
@@ -1182,6 +1208,7 @@ int main(int argc, char **argv) {
   test_metadata_plugin_success();
   test_metadata_plugin_failure();
   test_get_well_known_google_credentials_file_path();
+  test_channel_creds_duplicate_without_call_creds();
   grpc_shutdown();
   return 0;
 }

From 966a448a55fa64a5c4654b8cd9656b79cc4b5ba8 Mon Sep 17 00:00:00 2001
From: Dan Born <dborn@google.com>
Date: Mon, 26 Sep 2016 15:51:42 -0700
Subject: [PATCH 04/52] Require non-NULL exec_ctx to unref.

---
 src/core/lib/iomgr/tcp_server.h         |  4 ++--
 src/core/lib/iomgr/tcp_server_posix.c   | 11 -----------
 src/core/lib/iomgr/tcp_server_windows.c | 19 +++++++------------
 3 files changed, 9 insertions(+), 25 deletions(-)

diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index 5a25d39a0c4..9a390699b45 100644
--- a/src/core/lib/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -101,8 +101,8 @@ grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s);
 void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
                                            grpc_closure *shutdown_starting);
 
-/* If the refcount drops to zero, delete s, and call (exec_ctx==NULL) or enqueue
-   a call (exec_ctx!=NULL) to shutdown_complete. */
+/* If the refcount drops to zero, enqueue calls on exec_ctx to
+   shutdown_listeners and delete s. */
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s);
 
 /* Shutdown the fds of listeners. */
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index 5f846c8afb1..73df5477e66 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -745,22 +745,11 @@ void grpc_tcp_server_shutdown_starting_add(grpc_tcp_server *s,
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (gpr_unref(&s->refs)) {
-    grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
-    bool finish_ctx = false;
-    /* FIXME: API allows a NULL exec_ctx, although this might cause us to delete
-       ourself before some enqueued work in some other exec_ctx runs. */
-    if (exec_ctx == NULL) {
-      exec_ctx = &local_exec_ctx;
-      finish_ctx = true;
-    }
     grpc_tcp_server_shutdown_listeners(exec_ctx, s);
     gpr_mu_lock(&s->mu);
     grpc_exec_ctx_enqueue_list(exec_ctx, &s->shutdown_starting, NULL);
     gpr_mu_unlock(&s->mu);
     tcp_server_destroy(exec_ctx, s);
-    if (finish_ctx) {
-      grpc_exec_ctx_finish(exec_ctx);
-    }
   }
 }
 
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 1b125e7005b..35faded993d 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -121,6 +121,9 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
+  gpr_mu_lock(&s->mu);
+  GPR_ASSERT(s->shutdown);
+  gpr_mu_unlock(&s->mu);
   if (s->shutdown_complete != NULL) {
     grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }
@@ -139,7 +142,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 }
 
 grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
-  gpr_ref(&s->refs);
+  gpr_ref_non_zero(&s->refs);
   return s;
 }
 
@@ -174,19 +177,11 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   if (gpr_unref(&s->refs)) {
-    /* Complete shutdown_starting work before destroying. */
-    grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_tcp_server_shutdown_listeners(exec_ctx, s);
     gpr_mu_lock(&s->mu);
-    grpc_exec_ctx_enqueue_list(&local_exec_ctx, &s->shutdown_starting, NULL);
+    grpc_exec_ctx_enqueue_list(exec_ctx, &s->shutdown_starting, NULL);
     gpr_mu_unlock(&s->mu);
-    if (exec_ctx == NULL) {
-      grpc_exec_ctx_flush(&local_exec_ctx);
-      tcp_server_destroy(&local_exec_ctx, s);
-      grpc_exec_ctx_finish(&local_exec_ctx);
-    } else {
-      grpc_exec_ctx_finish(&local_exec_ctx);
-      tcp_server_destroy(exec_ctx, s);
-    }
+    tcp_server_destroy(exec_ctx, s);
   }
 }
 

From 42511cfd8b7c56c176c819311ea4dd4ade4df960 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Tue, 27 Sep 2016 18:15:54 -0700
Subject: [PATCH 05/52] Addressed review feedback

1. modified documentation
2. changed test slightly to make it more robust to accidental cache hits
---
 doc/interop-test-descriptions.md   | 27 +++++++++++++++++++--------
 test/cpp/interop/interop_client.cc |  9 +++++----
 test/cpp/interop/interop_server.cc |  2 +-
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 5b3ad2335cc..e3a41b12958 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -65,21 +65,21 @@ ensure that the proto serialized to zero bytes.*
 This test verifies that gRPC requests marked as cacheable use GET verb instead
 of POST, and that server sets appropriate cache control headers for the response
 to be cached by a proxy. This interop test requires that the server is behind
-a caching proxy. It is NOT expected to pass if client is accessing the server
-directly.
+a caching proxy. Use of current timestamp in the request prevents accidental
+cache matches left over from previous tests.
 
 Server features:
 * [CacheableUnaryCall][]
 
 Procedure:
- 1. Client calls CacheableUnaryCall twice, and compares the two responses.
- The server generates a unique response (timestamp) for each request.
- If the second response was delivered from cache, then both responses will
- be the same.
+ 1. Client calls CacheableUnaryCall with `SimpleRequest` request with payload
+    set to current timestamp.
+ 2. Client calls CacheableUnaryCall with `SimpleRequest` request again
+    immediately with the same payload as the previous request.
 
 Client asserts:
-* call was successful
-* responses are the same.
+* Both calls were successful
+* The payload body of both responses is the same.
 
 ### large_unary
 
@@ -962,6 +962,17 @@ payload body of size `SimpleRequest.response_size` bytes and type as appropriate
 for the `SimpleRequest.response_type`. If the server does not support the
 `response_type`, then it should fail the RPC with `INVALID_ARGUMENT`.
 
+### CacheableUnaryCall
+
+Server gets the default Empty proto as the request. It returns the
+SimpleResponse proto with the payload set to current timestamp string.
+In addition it adds
+  1. cache control headers such that the response can be cached by proxies in
+     the response path. Server should be behind a caching proxy for this test
+     to pass.
+  2. adds a `x-user-ip` header with `1.2.3.4` to the response. This is done
+     since some proxys such as GFE will not cache requests from localhost.
+
 ### CompressedResponse
 [CompressedResponse]: #compressedresponse
 
diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index f2290adfc3b..49ecf2620e2 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -848,10 +848,11 @@ bool InteropClient::DoStatusWithMessage() {
 bool InteropClient::DoCacheableUnary() {
   gpr_log(GPR_DEBUG, "Sending RPC with cacheable response");
 
+  // Create request with current timestamp
+  gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
+  std::string timestamp = std::to_string(ts.tv_nsec);
   SimpleRequest request;
-  request.set_response_size(16);
-  grpc::string payload(16, '\0');
-  request.mutable_payload()->set_body(payload.c_str(), 16);
+  request.mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
 
   // Request 1
   ClientContext context1;
@@ -878,7 +879,7 @@ bool InteropClient::DoCacheableUnary() {
   if (!AssertStatusOk(s2)) {
     return false;
   }
-  gpr_log(GPR_DEBUG, "response 1 payload: %s",
+  gpr_log(GPR_DEBUG, "response 2 payload: %s",
           response2.payload().body().c_str());
 
   // Check that the body is same for both requests. It will be the same if the
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index ac2567eba02..64eec4241af 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -159,7 +159,7 @@ class TestServiceImpl : public TestService::Service {
     gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
     std::string timestamp = std::to_string(ts.tv_nsec);
     response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
-    context->AddInitialMetadata("Cache-Control", "max-age=100000, public");
+    context->AddInitialMetadata("cache-control", "max-age=100000, public");
     return Status::OK;
   }
 

From a04c6789632c1e454cfe563f3144c27498d5ef0a Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 28 Sep 2016 08:43:00 -0700
Subject: [PATCH 06/52] trivial doc fix.

---
 doc/interop-test-descriptions.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index e3a41b12958..8a1e93eee02 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -964,9 +964,9 @@ for the `SimpleRequest.response_type`. If the server does not support the
 
 ### CacheableUnaryCall
 
-Server gets the default Empty proto as the request. It returns the
-SimpleResponse proto with the payload set to current timestamp string.
-In addition it adds
+Server gets the default SimpleRequest proto as the request. The content of the
+request are ignored. It returns the SimpleResponse proto with the payload set 
+to current timestamp string.  In addition it adds
   1. cache control headers such that the response can be cached by proxies in
      the response path. Server should be behind a caching proxy for this test
      to pass.

From 012fc18be93b98967a20986469eada34eac0c061 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 28 Sep 2016 10:46:27 -0700
Subject: [PATCH 07/52] doc fixes and max-age set to 60

---
 doc/interop-test-descriptions.md   | 15 +++++++++------
 test/cpp/interop/interop_server.cc |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 8a1e93eee02..62d36708f93 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -67,13 +67,17 @@ of POST, and that server sets appropriate cache control headers for the response
 to be cached by a proxy. This interop test requires that the server is behind
 a caching proxy. Use of current timestamp in the request prevents accidental
 cache matches left over from previous tests.
+Note that client adds a `x-user-ip` header with value `1.2.3.4` to the request.
+This is done since some proxys such as GFE will not cache requests from
+localhost.
 
 Server features:
 * [CacheableUnaryCall][]
 
 Procedure:
  1. Client calls CacheableUnaryCall with `SimpleRequest` request with payload
-    set to current timestamp.
+    set to current timestamp. Timestamp format is irrelevant, and resolution is
+    in nanoseconds.
  2. Client calls CacheableUnaryCall with `SimpleRequest` request again
     immediately with the same payload as the previous request.
 
@@ -965,13 +969,12 @@ for the `SimpleRequest.response_type`. If the server does not support the
 ### CacheableUnaryCall
 
 Server gets the default SimpleRequest proto as the request. The content of the
-request are ignored. It returns the SimpleResponse proto with the payload set 
-to current timestamp string.  In addition it adds
+request is ignored. It returns the SimpleResponse proto with the payload set
+to current timestamp.  The timestamp is an integer representing current time
+with nanosecond resolution. In addition it adds
   1. cache control headers such that the response can be cached by proxies in
      the response path. Server should be behind a caching proxy for this test
-     to pass.
-  2. adds a `x-user-ip` header with `1.2.3.4` to the response. This is done
-     since some proxys such as GFE will not cache requests from localhost.
+     to pass. Currently we set the max-age to 60 seconds.
 
 ### CompressedResponse
 [CompressedResponse]: #compressedresponse
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index 64eec4241af..06d1bdb7965 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -159,7 +159,7 @@ class TestServiceImpl : public TestService::Service {
     gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
     std::string timestamp = std::to_string(ts.tv_nsec);
     response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
-    context->AddInitialMetadata("cache-control", "max-age=100000, public");
+    context->AddInitialMetadata("cache-control", "max-age=60, public");
     return Status::OK;
   }
 

From ed3e86b7d907c0d09ce70d794cb964c5a4a2a53e Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 28 Sep 2016 10:55:49 -0700
Subject: [PATCH 08/52] added comment about setting cacheable flag.

---
 doc/interop-test-descriptions.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 62d36708f93..19947b3c60d 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -70,6 +70,10 @@ cache matches left over from previous tests.
 Note that client adds a `x-user-ip` header with value `1.2.3.4` to the request.
 This is done since some proxys such as GFE will not cache requests from
 localhost.
+Note also that the client request needs to marked as cacheable. For now this is
+achieved by setting the cacheable flag in the request context to 'true'.Longer
+term this will be automatically set via method options specified in the proto
+file.
 
 Server features:
 * [CacheableUnaryCall][]

From 1bb6e68fde7266c0ff626b4d0fcc735d1710d9c0 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 28 Sep 2016 11:16:15 -0700
Subject: [PATCH 09/52] more doc fixes

---
 doc/interop-test-descriptions.md | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 19947b3c60d..92824df23d6 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -67,13 +67,6 @@ of POST, and that server sets appropriate cache control headers for the response
 to be cached by a proxy. This interop test requires that the server is behind
 a caching proxy. Use of current timestamp in the request prevents accidental
 cache matches left over from previous tests.
-Note that client adds a `x-user-ip` header with value `1.2.3.4` to the request.
-This is done since some proxys such as GFE will not cache requests from
-localhost.
-Note also that the client request needs to marked as cacheable. For now this is
-achieved by setting the cacheable flag in the request context to 'true'.Longer
-term this will be automatically set via method options specified in the proto
-file.
 
 Server features:
 * [CacheableUnaryCall][]
@@ -82,8 +75,15 @@ Procedure:
  1. Client calls CacheableUnaryCall with `SimpleRequest` request with payload
     set to current timestamp. Timestamp format is irrelevant, and resolution is
     in nanoseconds.
+    Client adds a `x-user-ip` header with value `1.2.3.4` to the request.
+    This is done since some proxys such as GFE will not cache requests from
+    localhost.
+    Client marks the request as cacheable by setting the cacheable flag in the
+    request context. Longer term this should be driven by the method option
+    specified in the proto file itself.
  2. Client calls CacheableUnaryCall with `SimpleRequest` request again
-    immediately with the same payload as the previous request.
+    immediately with the same payload as the previous request. Cacheable flat is
+    also set for this request's context.
 
 Client asserts:
 * Both calls were successful
@@ -975,7 +975,9 @@ for the `SimpleRequest.response_type`. If the server does not support the
 Server gets the default SimpleRequest proto as the request. The content of the
 request is ignored. It returns the SimpleResponse proto with the payload set
 to current timestamp.  The timestamp is an integer representing current time
-with nanosecond resolution. In addition it adds
+with nanosecond resolution. This integer is formated as ASCII decimal in the
+response. The format is not really important as long as the response payload
+is different for each request. In addition it adds
   1. cache control headers such that the response can be cached by proxies in
      the response path. Server should be behind a caching proxy for this test
      to pass. Currently we set the max-age to 60 seconds.

From af564a1e920af86260a8003e20091c9eaa4e1c81 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 28 Sep 2016 12:50:37 -0700
Subject: [PATCH 10/52] changed timestamp clock from REALTIME to PRECISE

to increase robustness
---
 test/cpp/interop/interop_client.cc | 2 +-
 test/cpp/interop/interop_server.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index 49ecf2620e2..f323090ebf0 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -849,7 +849,7 @@ bool InteropClient::DoCacheableUnary() {
   gpr_log(GPR_DEBUG, "Sending RPC with cacheable response");
 
   // Create request with current timestamp
-  gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
+  gpr_timespec ts = gpr_now(GPR_CLOCK_PRECISE);
   std::string timestamp = std::to_string(ts.tv_nsec);
   SimpleRequest request;
   request.mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index 06d1bdb7965..e5e62dfc1ae 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -156,7 +156,7 @@ class TestServiceImpl : public TestService::Service {
   // Response contains current timestamp. We ignore everything in the request.
   Status CacheableUnaryCall(ServerContext* context, const SimpleRequest* request,
                    SimpleResponse* response) {
-    gpr_timespec ts = gpr_now(GPR_CLOCK_REALTIME);
+    gpr_timespec ts = gpr_now(GPR_CLOCK_PRECISE);
     std::string timestamp = std::to_string(ts.tv_nsec);
     response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
     context->AddInitialMetadata("cache-control", "max-age=60, public");

From befac97048f42f55409bb38a34a75c38cbf34241 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 22 Sep 2016 17:15:15 -0700
Subject: [PATCH 11/52] fixed local cloning of grpc/grpc submodules on docker

---
 test/distrib/cpp/run_distrib_test.sh                       | 6 +++++-
 .../grpc_check_generated_pb_files/check_pb_files.sh        | 6 +++++-
 .../interoptest/grpc_interop_csharp/build_interop.sh       | 6 +++++-
 .../grpc_interop_csharpcoreclr/build_interop.sh            | 6 +++++-
 .../interoptest/grpc_interop_cxx/build_interop.sh          | 6 +++++-
 .../interoptest/grpc_interop_http2/build_interop.sh        | 6 +++++-
 .../interoptest/grpc_interop_node/build_interop.sh         | 6 +++++-
 .../interoptest/grpc_interop_php/build_interop.sh          | 7 ++++++-
 .../interoptest/grpc_interop_php7/build_interop.sh         | 7 ++++++-
 .../interoptest/grpc_interop_python/build_interop.sh       | 6 +++++-
 .../interoptest/grpc_interop_ruby/build_interop.sh         | 7 ++++++-
 .../grpc_interop_stress_csharp/build_interop_stress.sh     | 6 +++++-
 .../grpc_interop_stress_cxx/build_interop_stress.sh        | 6 +++++-
 .../grpc_interop_stress_go/build_interop_stress.sh         | 6 +++++-
 .../grpc_interop_stress_java/build_interop_stress.sh       | 6 +++++-
 .../grpc_interop_stress_node/build_interop_stress.sh       | 6 +++++-
 .../grpc_interop_stress_php/build_interop_stress.sh        | 7 ++++++-
 .../grpc_interop_stress_python/build_interop_stress.sh     | 6 +++++-
 .../grpc_interop_stress_ruby/build_interop_stress.sh       | 7 ++++++-
 tools/run_tests/dockerize/docker_run.sh                    | 6 +++++-
 tools/run_tests/dockerize/docker_run_tests.sh              | 6 +++++-
 21 files changed, 110 insertions(+), 21 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index bc84b84b8f3..cd4158eb5d9 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -30,9 +30,13 @@
 
 set -ex
 
-git clone --recursive $EXTERNAL_GIT_ROOT
+git clone $EXTERNAL_GIT_ROOT
 cd grpc
 
+# clone submodules
+git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git \
+submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
+
 cd third_party/protobuf && ./autogen.sh && \
 ./configure && make -j4 && make check && make install && ldconfig
 
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index 62e41755ec1..fa9fd8fc72e 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -31,10 +31,14 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build grpc cpp plugin for generating grpc pb files
 make grpc_cpp_plugin
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index fd5436c44ff..4ec41b9c30a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index 77cd65cce37..073c4625bc3 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -32,12 +32,16 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 1c0828d23a6..5de62a41c0b 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 # build C++ interop client & server
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 46ddaf929a8..64418bc63de 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -32,7 +32,11 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 976f55d9ab5..85750ed91c2 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index a84a450221e..ea149049338 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 261dded2821..0e045631c2f 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index f29c59da8e8..55d1b6f915a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -32,11 +32,15 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 97b3860f981..0290de92099 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # build Ruby interop client and server
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index 1f4bf893cce..a4d7feef95f 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # Build C++ metrics client (to query the metrics from csharp stress client)
 make metrics_client -j
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index b67b1a1664a..d704f86496b 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 make install-certs
 
 BUILD_TYPE=${BUILD_TYPE:=opt}
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index 919d885c178..2b3b69be6ee 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -38,7 +38,11 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 
 # Clone the 'grpc' repo. We just need this for the wrapper scripts under
 # grpc/tools/gcp/stress_tests
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index d4fdfbbac96..99d287b21bf 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -36,7 +36,11 @@ mkdir -p /var/local/git
 git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java
 
 # grpc repo (for metrics client and for the stress test wrapper scripts)
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 976f55d9ab5..85750ed91c2 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 87262f1d629..997cda6a35a 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -ex
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 make install-certs
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index e65332f2f30..e3ccd8e00d3 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -32,13 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
 # Build c++ interop client
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index 1b7567d87a6..be39fb484b2 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -32,12 +32,17 @@
 set -e
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
+
+# clone gRPC submodules
+git submodule | awk '{ system("git submodule update --init --reference \
+./../../jenkins/grpc" $2 " " $2) }'
+
 rvm --default use ruby-2.1
 
 # Build Ruby interop client and server
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index f04b1cfb55b..bb875e275e7 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -36,7 +36,11 @@ set -ex
 if [ "$RELATIVE_COPY_PATH" == "" ]
 then
   mkdir -p /var/local/git
-  git clone --recursive "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  # clone gRPC submodules
+  (cd var/local/git/grpc && exec git submodule | awk -v \
+  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git submodule update --init \
+  --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"
   cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH"
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 8c6143d24f9..55371589605 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -42,7 +42,11 @@ export PATH=$PATH:/usr/bin/llvm-symbolizer
 chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
-git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+git clone /var/local/jenkins/grpc /var/local/git/grpc
+
+# clone gRPC submodules
+(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
+update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
 
 mkdir -p reports
 

From ce9471c962d95099e417ee7088376ecf8e017a78 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 22 Sep 2016 18:15:34 -0700
Subject: [PATCH 12/52] fixed wrong directory in git clone commands

---
 .../dockerfile/interoptest/grpc_interop_csharp/build_interop.sh | 2 +-
 .../interoptest/grpc_interop_csharpcoreclr/build_interop.sh     | 2 +-
 tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh  | 2 +-
 .../dockerfile/interoptest/grpc_interop_http2/build_interop.sh  | 2 +-
 tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh | 2 +-
 tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh  | 2 +-
 tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh | 2 +-
 .../dockerfile/interoptest/grpc_interop_python/build_interop.sh | 2 +-
 tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh | 2 +-
 .../grpc_interop_stress_csharp/build_interop_stress.sh          | 2 +-
 .../stress_test/grpc_interop_stress_cxx/build_interop_stress.sh | 2 +-
 .../stress_test/grpc_interop_stress_go/build_interop_stress.sh  | 2 +-
 .../grpc_interop_stress_java/build_interop_stress.sh            | 2 +-
 .../grpc_interop_stress_node/build_interop_stress.sh            | 2 +-
 .../stress_test/grpc_interop_stress_php/build_interop_stress.sh | 2 +-
 .../grpc_interop_stress_python/build_interop_stress.sh          | 2 +-
 .../grpc_interop_stress_ruby/build_interop_stress.sh            | 2 +-
 tools/run_tests/dockerize/docker_run_tests.sh                   | 2 +-
 18 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index 4ec41b9c30a..50378032dba 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index 073c4625bc3..c4092743034 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 5de62a41c0b..6f8b41241f9 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 make install-certs
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 64418bc63de..9c99bf56f54 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -36,7 +36,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # clone gRPC submodules
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
+update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 85750ed91c2..c82972d34e8 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # build Node interop client & server
 npm install -g node-gyp
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index ea149049338..fcdb28dcdf3 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 rvm --default use ruby-2.1
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 0e045631c2f..865802e30b6 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 rvm --default use ruby-2.1
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index 55d1b6f915a..917477c8471 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -41,6 +41,6 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 tools/run_tests/run_tests.py -l python -c opt --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 0290de92099..449a21ff650 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 rvm --default use ruby-2.1
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index a4d7feef95f..e25783a0b3b 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # Build C++ metrics client (to query the metrics from csharp stress client)
 make metrics_client -j
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index d704f86496b..d223653f92f 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 make install-certs
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index 2b3b69be6ee..a1d243d411d 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -42,7 +42,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # clone gRPC submodules
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
+update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index 99d287b21bf..4400e74be54 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -40,7 +40,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # clone gRPC submodules
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
+update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 85750ed91c2..c82972d34e8 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # build Node interop client & server
 npm install -g node-gyp
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 997cda6a35a..d1ac59fd917 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 rvm --default use ruby-2.1
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index e3ccd8e00d3..ee70c1f3a9a 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index be39fb484b2..5e47bafe859 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -41,7 +41,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 rvm --default use ruby-2.1
 
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 55371589605..ee4c1a5f05d 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -46,7 +46,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 # clone gRPC submodules
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc" $2 " " $2) }')
+update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
 mkdir -p reports
 

From 41a56ac8acec1576dbdbfdbcbbd0b8c4cea4f58e Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 22 Sep 2016 18:27:01 -0700
Subject: [PATCH 13/52] fixed one more incorrect directory for git clone

---
 .../dockerfile/grpc_check_generated_pb_files/check_pb_files.sh  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index fa9fd8fc72e..b112fb11bb6 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -37,7 +37,7 @@ cd /var/local/git/grpc
 
 # clone gRPC submodules
 git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc" $2 " " $2) }'
+./../../jenkins/grpc/" $2 " " $2) }'
 
 # build grpc cpp plugin for generating grpc pb files
 make grpc_cpp_plugin

From a436bab47f394c339b6e8978da3317ff94abfdf8 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 22 Sep 2016 18:49:24 -0700
Subject: [PATCH 14/52] fixed incorrect directory when using EXTERNAL_GIT_ROOT
 in Docker cloning var

---
 test/distrib/cpp/run_distrib_test.sh    | 2 +-
 tools/run_tests/dockerize/docker_run.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index cd4158eb5d9..9c8da508334 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -34,7 +34,7 @@ git clone $EXTERNAL_GIT_ROOT
 cd grpc
 
 # clone submodules
-git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git \
+git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git \
 submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
 
 cd third_party/protobuf && ./autogen.sh && \
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index bb875e275e7..791ed6d7b00 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -39,7 +39,7 @@ then
   git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
   # clone gRPC submodules
   (cd var/local/git/grpc && exec git submodule | awk -v \
-  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT '{ system("git submodule update --init \
+  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git submodule update --init \
   --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"

From 46c7f574bd4a5298f09fd5d72924b93c355e357e Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Fri, 23 Sep 2016 14:36:23 -0700
Subject: [PATCH 15/52] submodule clone comments changed to be more descriptive

---
 test/distrib/cpp/run_distrib_test.sh                            | 2 +-
 .../dockerfile/grpc_check_generated_pb_files/check_pb_files.sh  | 2 +-
 .../dockerfile/interoptest/grpc_interop_csharp/build_interop.sh | 2 +-
 .../interoptest/grpc_interop_csharpcoreclr/build_interop.sh     | 2 +-
 tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh  | 2 +-
 .../dockerfile/interoptest/grpc_interop_http2/build_interop.sh  | 2 +-
 tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh | 2 +-
 tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh  | 2 +-
 tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh | 2 +-
 .../dockerfile/interoptest/grpc_interop_python/build_interop.sh | 2 +-
 tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh | 2 +-
 .../grpc_interop_stress_csharp/build_interop_stress.sh          | 2 +-
 .../stress_test/grpc_interop_stress_cxx/build_interop_stress.sh | 2 +-
 .../stress_test/grpc_interop_stress_go/build_interop_stress.sh  | 2 +-
 .../grpc_interop_stress_java/build_interop_stress.sh            | 2 +-
 .../grpc_interop_stress_node/build_interop_stress.sh            | 2 +-
 .../stress_test/grpc_interop_stress_php/build_interop_stress.sh | 2 +-
 .../grpc_interop_stress_python/build_interop_stress.sh          | 2 +-
 .../grpc_interop_stress_ruby/build_interop_stress.sh            | 2 +-
 tools/run_tests/dockerize/docker_run.sh                         | 2 +-
 tools/run_tests/dockerize/docker_run_tests.sh                   | 2 +-
 21 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index 9c8da508334..6b2e043491f 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -33,7 +33,7 @@ set -ex
 git clone $EXTERNAL_GIT_ROOT
 cd grpc
 
-# clone submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git \
 submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
 
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index b112fb11bb6..4773a602a4c 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -35,7 +35,7 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index 50378032dba..6a0b2ef0605 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index c4092743034..c2bb64e84b5 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 6f8b41241f9..47380c8a125 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 9c99bf56f54..3adc8394902 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
 update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index c82972d34e8..4b73bb56f80 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index fcdb28dcdf3..48a6444821a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 865802e30b6..c2348f4f666 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index 917477c8471..e85153f51a0 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 449a21ff650..e5edb5335cb 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index e25783a0b3b..84920860803 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index d223653f92f..9ab7aa07274 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index a1d243d411d..de8430d6778 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -40,7 +40,7 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 # grpc/tools/gcp/stress_tests
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
 update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index 4400e74be54..ffffea18d21 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -38,7 +38,7 @@ git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc
 # grpc repo (for metrics client and for the stress test wrapper scripts)
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
 update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index c82972d34e8..4b73bb56f80 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index d1ac59fd917..78bf32ce519 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index ee70c1f3a9a..3f85a868032 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index 5e47bafe859..09b6fb1e117 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -39,7 +39,7 @@ cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 git submodule | awk '{ system("git submodule update --init --reference \
 ./../../jenkins/grpc/" $2 " " $2) }'
 
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index 791ed6d7b00..e57ceaf0ac8 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -37,7 +37,7 @@ if [ "$RELATIVE_COPY_PATH" == "" ]
 then
   mkdir -p /var/local/git
   git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
-  # clone gRPC submodules
+  # clone gRPC submodules, use data from locally cloned submodules where possible
   (cd var/local/git/grpc && exec git submodule | awk -v \
   EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git submodule update --init \
   --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index ee4c1a5f05d..a1e94b29527 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -44,7 +44,7 @@ chown $(whoami) $XDG_CACHE_HOME
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 
-# clone gRPC submodules
+# clone gRPC submodules, use data from locally cloned submodules where possible
 (cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
 update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
 

From 5d0f24600ef9b985bef3aada69e2f5847467c53c Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Mon, 26 Sep 2016 12:05:05 -0700
Subject: [PATCH 16/52] changed method to local clone submodules to using git
 submodule foreach from the local copy

---
 test/distrib/cpp/run_distrib_test.sh                   | 10 ++++++----
 .../grpc_check_generated_pb_files/check_pb_files.sh    |  8 ++++----
 .../interoptest/grpc_interop_csharp/build_interop.sh   |  8 ++++----
 .../grpc_interop_csharpcoreclr/build_interop.sh        |  8 ++++----
 .../interoptest/grpc_interop_cxx/build_interop.sh      |  8 ++++----
 .../interoptest/grpc_interop_http2/build_interop.sh    |  6 +++---
 .../interoptest/grpc_interop_node/build_interop.sh     |  8 ++++----
 .../interoptest/grpc_interop_php/build_interop.sh      |  8 ++++----
 .../interoptest/grpc_interop_php7/build_interop.sh     |  8 ++++----
 .../interoptest/grpc_interop_python/build_interop.sh   |  8 ++++----
 .../interoptest/grpc_interop_ruby/build_interop.sh     |  8 ++++----
 .../grpc_interop_stress_csharp/build_interop_stress.sh |  8 ++++----
 .../grpc_interop_stress_cxx/build_interop_stress.sh    |  8 ++++----
 .../grpc_interop_stress_go/build_interop_stress.sh     |  6 +++---
 .../grpc_interop_stress_java/build_interop_stress.sh   |  6 +++---
 .../grpc_interop_stress_node/build_interop_stress.sh   |  8 ++++----
 .../grpc_interop_stress_php/build_interop_stress.sh    |  8 ++++----
 .../grpc_interop_stress_python/build_interop_stress.sh |  8 ++++----
 .../grpc_interop_stress_ruby/build_interop_stress.sh   |  8 ++++----
 tools/run_tests/dockerize/docker_run.sh                |  6 +++---
 tools/run_tests/dockerize/docker_run_tests.sh          |  6 +++---
 21 files changed, 81 insertions(+), 79 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index 6b2e043491f..4c5deeaaae9 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -31,11 +31,13 @@
 set -ex
 
 git clone $EXTERNAL_GIT_ROOT
-cd grpc
-
 # clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk -v EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git \
-submodule update --init --reference " EXTERNAL_GIT_ROOT$2 " " $2) }'
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
+ 
+
+cd grpc
 
 cd third_party/protobuf && ./autogen.sh && \
 ./configure && make -j4 && make check && make install && ldconfig
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index 4773a602a4c..71306bfb676 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -32,13 +32,13 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # build grpc cpp plugin for generating grpc pb files
 make grpc_cpp_plugin
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index 6a0b2ef0605..8f7be9ba134 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -33,15 +33,15 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index c2bb64e84b5..476b69b09ae 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -33,15 +33,15 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # build C# interop client & server
 tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 47380c8a125..0822ed31052 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 make install-certs
 
 # build C++ interop client & server
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 3adc8394902..907ee6b3642 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -33,10 +33,10 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
-
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 4b73bb56f80..30dc8a67046 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index 48a6444821a..0fef66e512b 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -33,16 +33,16 @@ set -ex
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index c2348f4f666..5d891c6c0b3 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -33,16 +33,16 @@ set -ex
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index e85153f51a0..4ffd2a63c39 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -33,14 +33,14 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 tools/run_tests/run_tests.py -l python -c opt --build_only
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index e5edb5335cb..68aa7a1643a 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 rvm --default use ruby-2.1
 
 # build Ruby interop client and server
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index 84920860803..7f2d587e59a 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # Build C++ metrics client (to query the metrics from csharp stress client)
 make metrics_client -j
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index 9ab7aa07274..7332a8c1bbe 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 make install-certs
 
 BUILD_TYPE=${BUILD_TYPE:=opt}
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index de8430d6778..f0a54a43626 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -39,10 +39,10 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 # Clone the 'grpc' repo. We just need this for the wrapper scripts under
 # grpc/tools/gcp/stress_tests
 git clone /var/local/jenkins/grpc /var/local/git/grpc
-
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index ffffea18d21..dcd3a0644a2 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -37,10 +37,10 @@ git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc
 
 # grpc repo (for metrics client and for the stress test wrapper scripts)
 git clone /var/local/jenkins/grpc /var/local/git/grpc
-
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 4b73bb56f80..30dc8a67046 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 # build Node interop client & server
 npm install -g node-gyp
 npm install --unsafe-perm --build-from-source
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 78bf32ce519..0f624bf3217 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -ex
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 rvm --default use ruby-2.1
 
 make install-certs
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index 3f85a868032..57b916ca4dd 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
 # Build c++ interop client
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index 09b6fb1e117..bb4d2fcd224 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -33,16 +33,16 @@ set -e
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 # Copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
 
-# clone gRPC submodules, use data from locally cloned submodules where possible
-git submodule | awk '{ system("git submodule update --init --reference \
-./../../jenkins/grpc/" $2 " " $2) }'
-
 rvm --default use ruby-2.1
 
 # Build Ruby interop client and server
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index e57ceaf0ac8..d50ea81750b 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -38,9 +38,9 @@ then
   mkdir -p /var/local/git
   git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
   # clone gRPC submodules, use data from locally cloned submodules where possible
-  (cd var/local/git/grpc && exec git submodule | awk -v \
-  EXTERNAL_GIT_ROOT=$EXTERNAL_GIT_ROOT/ '{ system("git submodule update --init \
-  --reference " EXTERNAL_GIT_ROOT$2 " " $2) }')
+  (cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+  && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+  ${name}')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"
   cp -r "$EXTERNAL_GIT_ROOT/$RELATIVE_COPY_PATH"/* "/var/local/git/grpc/$RELATIVE_COPY_PATH"
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index a1e94b29527..41d93e444d4 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -43,10 +43,10 @@ chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
-
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/git/grpc/ && exec git submodule | awk '{ system("git submodule \
-update --init --reference ./../../jenkins/grpc/" $2 " " $2) }')
+(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 mkdir -p reports
 

From 10dcccadfb1400bdc016ad2861b601a9d2574d04 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Wed, 28 Sep 2016 16:52:34 -0700
Subject: [PATCH 17/52] fixed whitespacing and docker_run_tests.sh now clones
 submodule via network

---
 test/distrib/cpp/run_distrib_test.sh           |  5 ++---
 .../check_pb_files.sh                          |  2 +-
 .../grpc_interop_csharp/build_interop.sh       |  2 +-
 .../build_interop.sh                           |  2 +-
 .../grpc_interop_cxx/build_interop.sh          |  2 +-
 .../grpc_interop_http2/build_interop.sh        |  2 +-
 .../grpc_interop_node/build_interop.sh         |  2 +-
 .../grpc_interop_php/build_interop.sh          |  3 +--
 .../grpc_interop_php7/build_interop.sh         |  3 +--
 .../grpc_interop_python/build_interop.sh       |  2 +-
 .../grpc_interop_ruby/build_interop.sh         |  3 +--
 .../build_interop_stress.sh                    |  2 +-
 .../build_interop_stress.sh                    |  2 +-
 .../build_interop_stress.sh                    |  2 +-
 .../build_interop_stress.sh                    |  2 +-
 .../build_interop_stress.sh                    |  2 +-
 .../build_interop_stress.sh                    |  3 +--
 .../build_interop_stress.sh                    |  3 +--
 .../build_interop_stress.sh                    |  3 +--
 tools/run_tests/dockerize/docker_run.sh        |  6 +++---
 tools/run_tests/dockerize/docker_run_tests.sh  | 18 ++++++++++++++----
 21 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh
index 4c5deeaaae9..15fbf281074 100755
--- a/test/distrib/cpp/run_distrib_test.sh
+++ b/test/distrib/cpp/run_distrib_test.sh
@@ -32,10 +32,9 @@ set -ex
 
 git clone $EXTERNAL_GIT_ROOT
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+(cd ${EXTERNAL_GIT_ROOT} && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference ${EXTERNAL_GIT_ROOT}/${name} \
 ${name}')
- 
 
 cd grpc
 
diff --git a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
index 71306bfb676..9db7aae9eba 100755
--- a/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
+++ b/tools/dockerfile/grpc_check_generated_pb_files/check_pb_files.sh
@@ -33,7 +33,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
index 8f7be9ba134..e37070904d3 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharp/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
index 476b69b09ae..d90c899569f 100755
--- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
index 0822ed31052..7a7ca0d3d1c 100755
--- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
index 907ee6b3642..a1d668d69fc 100755
--- a/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
index 30dc8a67046..4116f842ff1 100755
--- a/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_node/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
index 0fef66e512b..ea2f88f2d2d 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh
@@ -34,7 +34,7 @@ set -ex
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
index 5d891c6c0b3..259b7e09750 100755
--- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -34,7 +34,7 @@ set -ex
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 rvm --default use ruby-2.1
 
 # gRPC core and protobuf need to be installed
diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
index 4ffd2a63c39..a6f8c7bfe6b 100755
--- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
index 68aa7a1643a..6cd2fa2b147 100755
--- a/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 rvm --default use ruby-2.1
 
 # build Ruby interop client and server
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
index 7f2d587e59a..345196894ef 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_csharp/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
index 7332a8c1bbe..92d1f80fe60 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_cxx/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
index f0a54a43626..9e4769cf334 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/build_interop_stress.sh
@@ -40,7 +40,7 @@ git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 # grpc/tools/gcp/stress_tests
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
index dcd3a0644a2..0194860d101 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_java/build_interop_stress.sh
@@ -38,7 +38,7 @@ git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc
 # grpc repo (for metrics client and for the stress test wrapper scripts)
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
index 30dc8a67046..4116f842ff1 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_node/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
index 0f624bf3217..c6d03ea28e0 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -ex
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 rvm --default use ruby-2.1
 
 make install-certs
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
index 57b916ca4dd..1c7dc2bd577 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 tools/run_tests/run_tests.py -l python -c opt --build_only
 
 # Build c++ interop client
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
index bb4d2fcd224..019f0a44e4c 100755
--- a/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_ruby/build_interop_stress.sh
@@ -34,7 +34,7 @@ set -e
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
 # clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
 && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
 ${name}')
 
@@ -42,7 +42,6 @@ ${name}')
 cp -r /var/local/jenkins/service_account $HOME || true
 
 cd /var/local/git/grpc
-
 rvm --default use ruby-2.1
 
 # Build Ruby interop client and server
diff --git a/tools/run_tests/dockerize/docker_run.sh b/tools/run_tests/dockerize/docker_run.sh
index d50ea81750b..ee8d288fb2a 100755
--- a/tools/run_tests/dockerize/docker_run.sh
+++ b/tools/run_tests/dockerize/docker_run.sh
@@ -36,10 +36,10 @@ set -ex
 if [ "$RELATIVE_COPY_PATH" == "" ]
 then
   mkdir -p /var/local/git
-  git clone "$EXTERNAL_GIT_ROOT" /var/local/git/grpc
+  git clone $EXTERNAL_GIT_ROOT /var/local/git/grpc
   # clone gRPC submodules, use data from locally cloned submodules where possible
-  (cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
-  && git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+  (cd ${EXTERNAL_GIT_ROOT} && git submodule foreach 'cd /var/local/git/grpc \
+  && git submodule update --init --reference ${EXTERNAL_GIT_ROOT}/${name} \
   ${name}')
 else
   mkdir -p "/var/local/git/grpc/$RELATIVE_COPY_PATH"
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 41d93e444d4..3bab7ffd903 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -43,10 +43,20 @@ chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
 git clone /var/local/jenkins/grpc /var/local/git/grpc
-# clone gRPC submodules, use data from locally cloned submodules where possible
-(cd /var/local/jenkins/grpc / && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
+
+# (todo (mattkwong): the /var/local/jenkins/grpc has no submodules and boringssl 
+# has non-submodule files in it. Figure out how to fix this for local cloning 
+## this prints to console "reports reports.zip"
+# ls /var/local/jenkins/grpc/third_party/boringssl
+## none of these print anything (empty directory)
+# ls /var/local/jenkins/grpc/third_party/gflags
+# ls /var/local/jenkins/grpc/third_party/googletest
+# ls /var/local/jenkins/grpc/third_party/nanopb
+# ls /var/local/jenkins/grpc/third_party/protobuf
+# ls /var/local/jenkins/grpc/third_party/thrift
+# ls /var/local/jenkins/grpc/third_party/zlib
+
+(cd /var/local/git/grpc && git submodule update --init --recursive )
 
 mkdir -p reports
 

From 66a1eea681b1337340e33abd211a8fb1233e8eab Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Thu, 29 Sep 2016 18:30:19 -0700
Subject: [PATCH 18/52] changed wheezy dockerfile to install git version > 1.7
 to support local submodule cloning

---
 .../dockerfile/test/cxx_wheezy_x64/Dockerfile |  7 +++++++
 tools/run_tests/dockerize/docker_run_tests.sh | 20 +++++--------------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
index c25033387f9..503a4357070 100644
--- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
@@ -63,6 +63,13 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+#================
+# Add backport to Debian sources.list and update git to version > 1.7
+RUN echo "deb http://http.debian.net/debian wheezy-backports main" \
+  >/etc/apt/sources.list.d/wheezy-backports.list
+RUN apt-get update -qq
+RUN apt-get -t wheezy-backports install -y -qq git mercurial
+
 #====================
 # Python dependencies
 
diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 3bab7ffd903..7108964a9bc 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -42,21 +42,11 @@ export PATH=$PATH:/usr/bin/llvm-symbolizer
 chown $(whoami) $XDG_CACHE_HOME
 
 mkdir -p /var/local/git
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-
-# (todo (mattkwong): the /var/local/jenkins/grpc has no submodules and boringssl 
-# has non-submodule files in it. Figure out how to fix this for local cloning 
-## this prints to console "reports reports.zip"
-# ls /var/local/jenkins/grpc/third_party/boringssl
-## none of these print anything (empty directory)
-# ls /var/local/jenkins/grpc/third_party/gflags
-# ls /var/local/jenkins/grpc/third_party/googletest
-# ls /var/local/jenkins/grpc/third_party/nanopb
-# ls /var/local/jenkins/grpc/third_party/protobuf
-# ls /var/local/jenkins/grpc/third_party/thrift
-# ls /var/local/jenkins/grpc/third_party/zlib
-
-(cd /var/local/git/grpc && git submodule update --init --recursive )
+git clone  /var/local/jenkins/grpc /var/local/git/grpc
+# clone gRPC submodules, use data from locally cloned submodules where possible
+(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
+&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
+${name}')
 
 mkdir -p reports
 

From b6cf4944a5f4fda11f240b3c1b9736c61d299e5b Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Fri, 30 Sep 2016 10:49:57 -0700
Subject: [PATCH 19/52] s/flat/flag

---
 doc/interop-test-descriptions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 92824df23d6..97d76191a8f 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -82,7 +82,7 @@ Procedure:
     request context. Longer term this should be driven by the method option
     specified in the proto file itself.
  2. Client calls CacheableUnaryCall with `SimpleRequest` request again
-    immediately with the same payload as the previous request. Cacheable flat is
+    immediately with the same payload as the previous request. Cacheable flag is
     also set for this request's context.
 
 Client asserts:

From c9beacadb1f9b0cd8858ce59d6cbed01b7f48cd3 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Fri, 30 Sep 2016 11:26:13 -0700
Subject: [PATCH 20/52] fix  for gcc 4.4 warning

---
 test/cpp/interop/interop_server.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index e5e62dfc1ae..b58b744b920 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -157,7 +157,7 @@ class TestServiceImpl : public TestService::Service {
   Status CacheableUnaryCall(ServerContext* context, const SimpleRequest* request,
                    SimpleResponse* response) {
     gpr_timespec ts = gpr_now(GPR_CLOCK_PRECISE);
-    std::string timestamp = std::to_string(ts.tv_nsec);
+    std::string timestamp = std::to_string((long long unsigned)ts.tv_nsec);
     response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
     context->AddInitialMetadata("cache-control", "max-age=60, public");
     return Status::OK;

From b58c2db6167cee26327e611ffccba499b8bd7015 Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Fri, 30 Sep 2016 16:21:11 -0700
Subject: [PATCH 21/52] yet another gcc 4.4 compile fix.

---
 test/cpp/interop/interop_client.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index f323090ebf0..2fbd6a98cdc 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -850,7 +850,7 @@ bool InteropClient::DoCacheableUnary() {
 
   // Create request with current timestamp
   gpr_timespec ts = gpr_now(GPR_CLOCK_PRECISE);
-  std::string timestamp = std::to_string(ts.tv_nsec);
+  std::string timestamp = std::to_string((long long unsigned)ts.tv_nsec);
   SimpleRequest request;
   request.mutable_payload()->set_body(timestamp.c_str(), timestamp.size());
 

From 8725870c5822a8d2f96e275e46475594567d5ccd Mon Sep 17 00:00:00 2001
From: Dan Born <dborn@google.com>
Date: Mon, 3 Oct 2016 12:46:16 -0700
Subject: [PATCH 22/52] Fix Windows server

---
 src/core/lib/iomgr/tcp_server_windows.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 35faded993d..4ff05601fa8 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -121,9 +121,6 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
-  gpr_mu_lock(&s->mu);
-  GPR_ASSERT(s->shutdown);
-  gpr_mu_unlock(&s->mu);
   if (s->shutdown_complete != NULL) {
     grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
   }

From 84350e167d4bc4f7c96af71baf28b5a5a0807856 Mon Sep 17 00:00:00 2001
From: Nathaniel Manista <nathaniel@google.com>
Date: Tue, 4 Oct 2016 20:25:53 +0000
Subject: [PATCH 23/52] Drop unnecessary return statement

---
 src/python/grpcio/grpc/_server.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py
index 94a13bfb2fb..f70cd2afa55 100644
--- a/src/python/grpcio/grpc/_server.py
+++ b/src/python/grpcio/grpc/_server.py
@@ -462,7 +462,6 @@ def _unary_response_in_pool(
           rpc_event, state, response, response_serializer)
       if serialized_response is not None:
         _status(rpc_event, state, serialized_response)
-  return
 
 
 def _stream_response_in_pool(

From 3b0d092e22a30d449952be210b174cb9d080b462 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Mon, 19 Sep 2016 14:33:18 +0200
Subject: [PATCH 24/52] add matrix run script

---
 tools/run_tests/run_tests_in_workspace.sh |  44 ++++
 tools/run_tests/run_tests_matrix.py       | 239 ++++++++++++++++++++++
 2 files changed, 283 insertions(+)
 create mode 100755 tools/run_tests/run_tests_in_workspace.sh
 create mode 100755 tools/run_tests/run_tests_matrix.py

diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/run_tests_in_workspace.sh
new file mode 100755
index 00000000000..0e7604dbc55
--- /dev/null
+++ b/tools/run_tests/run_tests_in_workspace.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Create a workspace in a subdirectory to allow running multiple builds in isolation.
+# WORKSPACE_NAME env variable needs to contain name of the workspace to create.
+# All cmdline args will be passed to run_tests.py script (executed in the 
+# newly created workspace)
+set -ex
+
+cd $(dirname $0)/../..
+
+rm -rf "${WORKSPACE_NAME}"
+git clone --recursive . "${WORKSPACE_NAME}"
+
+echo "Running run_tests.py in workspace ${WORKSPACE_NAME}" 
+"${WORKSPACE_NAME}/tools/run_tests/run_tests.py" $@
+
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
new file mode 100755
index 00000000000..2344e2f9fd8
--- /dev/null
+++ b/tools/run_tests/run_tests_matrix.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python2.7
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Run test matrix."""
+
+import argparse
+import jobset
+import os
+import report_utils
+import sys
+
+_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
+os.chdir(_ROOT)
+
+# TODO(jtattermusch): this is not going to be enough for sanitizers.
+_RUNTESTS_TIMEOUT = 30*60
+
+
+def _docker_jobspec(name, runtests_args=[]):
+  """Run a single instance of run_tests.py in a docker container"""
+  # TODO: fix copying report files from inside docker....
+  test_job = jobset.JobSpec(
+          cmdline=['python', 'tools/run_tests/run_tests.py',
+                   '--use_docker',
+                   '-t',
+                   '-j', '3',
+                   '-x', 'report_%s.xml' % name] + runtests_args,
+          shortname='run_tests_%s' % name,
+          timeout_seconds=_RUNTESTS_TIMEOUT)
+  return test_job
+
+
+def _workspace_jobspec(name, runtests_args=[], workspace_name=None):
+  """Run a single instance of run_tests.py in a separate workspace"""
+  env = {'WORKSPACE_NAME': workspace_name}
+  test_job = jobset.JobSpec(
+          cmdline=['tools/run_tests/run_tests_in_workspace.sh',
+                   '-t',
+                   '-j', '3',
+                   '-x', '../report_%s.xml' % name] + runtests_args,
+          environ=env,
+          shortname='run_tests_%s' % name,
+          timeout_seconds=_RUNTESTS_TIMEOUT)
+  return test_job
+
+
+def _generate_jobs(languages, configs, platforms,
+                  arch=None, compiler=None,
+                  labels=[]):
+  result = []
+  for language in languages:
+    for platform in platforms:
+      for config in configs:
+        name = '%s_%s_%s' % (language, platform, config)
+        runtests_args = ['-l', language,
+                         '-c', config]
+        if arch or compiler:
+          name += '_%s_%s' % (arch, compiler)
+          runtests_args += ['--arch', arch,
+                            '--compiler', compiler]
+
+        if platform == 'linux':
+          job = _docker_jobspec(name=name, runtests_args=runtests_args)
+        else:
+          job = _workspace_jobspec(name=name, runtests_args=runtests_args)
+
+        job.labels = [platform, config, language] + labels
+        result.append(job)
+  return result
+
+
+def _create_test_jobs():
+  test_jobs = []
+  # supported on linux only
+  test_jobs += _generate_jobs(languages=['sanity', 'php7'],
+                             configs=['dbg', 'opt'],
+                             platforms=['linux'],
+                             labels=['basictests'])
+  
+  # supported on all platforms.
+  test_jobs += _generate_jobs(languages=['c', 'csharp', 'node', 'python'],
+                            configs=['dbg', 'opt'],
+                            platforms=['linux', 'macos', 'windows'],
+                            labels=['basictests'])
+  
+  # supported on linux and mac.
+  test_jobs += _generate_jobs(languages=['c++', 'ruby', 'php'],
+                             configs=['dbg', 'opt'],
+                             platforms=['linux', 'macos'],
+                             labels=['basictests'])
+  
+  # supported on mac only.
+  test_jobs += _generate_jobs(languages=['objc'],
+                              configs=['dbg', 'opt'],
+                              platforms=['macos'],
+                              labels=['basictests'])
+  
+  # sanitizers
+  test_jobs += _generate_jobs(languages=['c'],
+                              configs=['msan', 'asan', 'tsan'],
+                              platforms=['linux'],
+                              labels=['sanitizers'])
+  test_jobs += _generate_jobs(languages=['c++'],
+                              configs=['asan', 'tsan'],
+                              platforms=['linux'],
+                              labels=['sanitizers'])
+  return test_jobs
+
+  
+def _create_portability_test_jobs():
+  test_jobs = []
+  # portability C x86
+  test_jobs += _generate_jobs(languages=['c'],
+                              configs=['dbg'],
+                              platforms=['linux'],
+                              arch='x86',
+                              compiler='default',
+                              labels=['portability'])
+  
+  # portability C and C++ on x64
+  for compiler in ['gcc4.4', 'gcc4.6', 'gcc5.3',
+                   'clang3.5', 'clang3.6', 'clang3.7']:
+    test_jobs += _generate_jobs(languages=['c', 'c++'],
+                                configs=['dbg'],
+                                platforms=['linux'],
+                                arch='x64',
+                                compiler=compiler,
+                                labels=['portability'])
+  
+  # portability C on Windows
+  for arch in ['x86', 'x64']:
+    for compiler in ['vs2013', 'vs2015']:
+      test_jobs += _generate_jobs(languages=['c'],
+                                  configs=['dbg'],
+                                  platforms=['windows'],
+                                  arch=arch,
+                                  compiler=compiler,
+                                  labels=['portability'])
+  
+  test_jobs += _generate_jobs(languages=['python'],
+                              configs=['dbg'],
+                              platforms=['linux'],
+                              arch='default',
+                              compiler='python3.4',
+                              labels=['portability'])
+  
+  test_jobs += _generate_jobs(languages=['csharp'],
+                              configs=['dbg'],
+                              platforms=['linux'],
+                              arch='default',
+                              compiler='coreclr',
+                              labels=['portability'])
+  
+  for compiler in ['node5', 'node6', 'node0.12']:
+    test_jobs += _generate_jobs(languages=['node'],
+                               configs=['dbg'],
+                               platforms=['linux'],
+                               arch='default',
+                               compiler=compiler,
+                               labels=['portability'])
+  return test_jobs  
+
+
+all_jobs = _create_test_jobs() + _create_portability_test_jobs()
+
+all_labels = set()
+for job in all_jobs:
+  for label in job.labels:
+    all_labels.add(label)
+
+argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
+argp.add_argument('-f', '--filter',
+                  choices=sorted(all_labels),
+                  nargs='+',
+                  default=[],
+                  help='Filter targets to run by label with AND semantics.')
+args = argp.parse_args()
+
+jobs = []
+for job in all_jobs:
+  if not args.filter or all(filter in job.labels for filter in args.filter):
+    jobs.append(job)
+
+if not jobs:
+  jobset.message('FAILED', 'No test suites match given criteria.',
+                 do_newline=True)
+  sys.exit(1)
+  
+print('IMPORTANT: The changes you are testing need to be locally committed')
+print('because only the committed changes in the current branch will be')
+print('copied to the docker environment or into subworkspaces.')
+
+print 
+print 'Will run these tests:'
+for job in jobs:
+  print '  %s' % job.shortname
+print
+
+jobset.message('START', 'Running test matrix.', do_newline=True)
+num_failures, resultset = jobset.run(jobs,
+                                     newline_on_success=True,
+                                     travis=True,
+                                     maxjobs=2)
+report_utils.render_junit_xml_report(resultset, 'report.xml')
+
+if num_failures == 0:
+  jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
+                 do_newline=True)
+else:
+  jobset.message('FAILED', 'Some run_tests.py instance have failed.',
+                 do_newline=True)
+  sys.exit(1)

From 0b19470d2b6ec821aa504ce615ba9f9d68596d28 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Mon, 19 Sep 2016 18:20:37 +0200
Subject: [PATCH 25/52] improve report collection

---
 tools/run_tests/dockerize/docker_run_tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/run_tests/dockerize/docker_run_tests.sh b/tools/run_tests/dockerize/docker_run_tests.sh
index 8c6143d24f9..ef02d266253 100755
--- a/tools/run_tests/dockerize/docker_run_tests.sh
+++ b/tools/run_tests/dockerize/docker_run_tests.sh
@@ -63,6 +63,7 @@ echo '</body></html>' >> index.html
 cd ..
 
 zip -r reports.zip reports
-find . -name report.xml | xargs zip reports.zip
+find . -name report.xml | xargs -r zip reports.zip
+find . -name 'report_*.xml' | xargs -r zip reports.zip
 
 exit $exit_code

From 7a7792a9de680c9d6925b092d5efbf17984feec5 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Mon, 19 Sep 2016 18:37:17 +0200
Subject: [PATCH 26/52] run matrix improvements

---
 .../dockerize/build_docker_and_run_tests.sh     | 12 ++++++------
 tools/run_tests/run_tests_in_workspace.sh       |  2 ++
 tools/run_tests/run_tests_matrix.py             | 17 ++++++++++++++---
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index c2ea6f2c6eb..b4b172ddef6 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -44,9 +44,6 @@ mkdir -p /tmp/ccache
 # its cache location now that --download-cache is deprecated).
 mkdir -p /tmp/xdg-cache-home
 
-# Create a local branch so the child Docker script won't complain
-git branch -f jenkins-docker
-
 # Inputs
 # DOCKERFILE_DIR - Directory in which Dockerfile file is located.
 # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root)
@@ -86,9 +83,12 @@ docker run \
   $DOCKER_IMAGE_NAME \
   bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true"
 
-docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" $git_root || true
-unzip -o $git_root/reports.zip -d $git_root || true
-rm -f reports.zip
+# use unique name for reports.zip to prevent clash between concurrent
+# run_tests.py runs 
+TEMP_REPORTS_ZIP=`mktemp`
+docker cp "$CONTAINER_NAME:/var/local/git/grpc/reports.zip" ${TEMP_REPORTS_ZIP} || true
+unzip -o ${TEMP_REPORTS_ZIP} -d $git_root || true
+rm -f ${TEMP_REPORTS_ZIP}
 
 # remove the container, possibly killing it first
 docker rm -f $CONTAINER_NAME || true
diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/run_tests_in_workspace.sh
index 0e7604dbc55..b0c9ad05f76 100755
--- a/tools/run_tests/run_tests_in_workspace.sh
+++ b/tools/run_tests/run_tests_in_workspace.sh
@@ -37,6 +37,8 @@ set -ex
 cd $(dirname $0)/../..
 
 rm -rf "${WORKSPACE_NAME}"
+# TODO(jtattermusch): clone --recursive fetches the submodules from github.
+# Try avoiding that to save time and network capacity.
 git clone --recursive . "${WORKSPACE_NAME}"
 
 echo "Running run_tests.py in workspace ${WORKSPACE_NAME}" 
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 2344e2f9fd8..bc2e37abd89 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -32,6 +32,7 @@
 
 import argparse
 import jobset
+import multiprocessing
 import os
 import report_utils
 import sys
@@ -40,8 +41,12 @@ _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(_ROOT)
 
 # TODO(jtattermusch): this is not going to be enough for sanitizers.
+# TODO(jtattermusch): this is not going to be enough for rebuilding clang docker.
 _RUNTESTS_TIMEOUT = 30*60
 
+# Number of jobs assigned to each run_tests.py instance
+_INNER_JOBS = 2
+
 
 def _docker_jobspec(name, runtests_args=[]):
   """Run a single instance of run_tests.py in a docker container"""
@@ -50,7 +55,7 @@ def _docker_jobspec(name, runtests_args=[]):
           cmdline=['python', 'tools/run_tests/run_tests.py',
                    '--use_docker',
                    '-t',
-                   '-j', '3',
+                   '-j', str(_INNER_JOBS),
                    '-x', 'report_%s.xml' % name] + runtests_args,
           shortname='run_tests_%s' % name,
           timeout_seconds=_RUNTESTS_TIMEOUT)
@@ -59,11 +64,13 @@ def _docker_jobspec(name, runtests_args=[]):
 
 def _workspace_jobspec(name, runtests_args=[], workspace_name=None):
   """Run a single instance of run_tests.py in a separate workspace"""
+  if not workspace_name:
+    workspace_name = 'workspace_%s' % name
   env = {'WORKSPACE_NAME': workspace_name}
   test_job = jobset.JobSpec(
           cmdline=['tools/run_tests/run_tests_in_workspace.sh',
                    '-t',
-                   '-j', '3',
+                   '-j', str(_INNER_JOBS),
                    '-x', '../report_%s.xml' % name] + runtests_args,
           environ=env,
           shortname='run_tests_%s' % name,
@@ -196,6 +203,10 @@ for job in all_jobs:
     all_labels.add(label)
 
 argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
+argp.add_argument('-j', '--jobs',
+                  default=multiprocessing.cpu_count()/_INNER_JOBS,
+                  type=int,
+                  help='Number of concurrent run_tests.py instances.')
 argp.add_argument('-f', '--filter',
                   choices=sorted(all_labels),
                   nargs='+',
@@ -227,7 +238,7 @@ jobset.message('START', 'Running test matrix.', do_newline=True)
 num_failures, resultset = jobset.run(jobs,
                                      newline_on_success=True,
                                      travis=True,
-                                     maxjobs=2)
+                                     maxjobs=args.jobs)
 report_utils.render_junit_xml_report(resultset, 'report.xml')
 
 if num_failures == 0:

From 5b34c4fc099d8b8df0696c1255009c144bf03cd4 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 20 Sep 2016 16:06:13 +0200
Subject: [PATCH 27/52] address some TODOs

---
 tools/run_tests/run_tests_matrix.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index bc2e37abd89..06ab02e32fb 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -40,9 +40,9 @@ import sys
 _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(_ROOT)
 
-# TODO(jtattermusch): this is not going to be enough for sanitizers.
-# TODO(jtattermusch): this is not going to be enough for rebuilding clang docker.
-_RUNTESTS_TIMEOUT = 30*60
+# Set the timeout high to allow enough time for sanitizers and pre-building
+# clang docker.
+_RUNTESTS_TIMEOUT = 2*60*60
 
 # Number of jobs assigned to each run_tests.py instance
 _INNER_JOBS = 2
@@ -50,7 +50,6 @@ _INNER_JOBS = 2
 
 def _docker_jobspec(name, runtests_args=[]):
   """Run a single instance of run_tests.py in a docker container"""
-  # TODO: fix copying report files from inside docker....
   test_job = jobset.JobSpec(
           cmdline=['python', 'tools/run_tests/run_tests.py',
                    '--use_docker',
@@ -203,6 +202,8 @@ for job in all_jobs:
     all_labels.add(label)
 
 argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
+# TODO(jtattermusch): allow running tests with --build_only flag
+# TODO(jtattermusch): allow running tests with --force_default_poller flag.
 argp.add_argument('-j', '--jobs',
                   default=multiprocessing.cpu_count()/_INNER_JOBS,
                   type=int,

From 7a6c2235d522b67e9dfe77311e2239463abc8cd5 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 22 Sep 2016 13:40:48 +0200
Subject: [PATCH 28/52] support passing extra args to run_tests.py

---
 tools/run_tests/run_tests_matrix.py | 90 +++++++++++++++++++----------
 1 file changed, 59 insertions(+), 31 deletions(-)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 06ab02e32fb..ae7fdd84f9b 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -79,7 +79,7 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None):
 
 def _generate_jobs(languages, configs, platforms,
                   arch=None, compiler=None,
-                  labels=[]):
+                  labels=[], extra_args=[]):
   result = []
   for language in languages:
     for platform in platforms:
@@ -92,6 +92,7 @@ def _generate_jobs(languages, configs, platforms,
           runtests_args += ['--arch', arch,
                             '--compiler', compiler]
 
+        runtests_args += extra_args
         if platform == 'linux':
           job = _docker_jobspec(name=name, runtests_args=runtests_args)
         else:
@@ -102,45 +103,51 @@ def _generate_jobs(languages, configs, platforms,
   return result
 
 
-def _create_test_jobs():
+def _create_test_jobs(extra_args=[]):
   test_jobs = []
   # supported on linux only
   test_jobs += _generate_jobs(languages=['sanity', 'php7'],
                              configs=['dbg', 'opt'],
                              platforms=['linux'],
-                             labels=['basictests'])
+                             labels=['basictests'],
+                             extra_args=extra_args)
   
   # supported on all platforms.
   test_jobs += _generate_jobs(languages=['c', 'csharp', 'node', 'python'],
-                            configs=['dbg', 'opt'],
-                            platforms=['linux', 'macos', 'windows'],
-                            labels=['basictests'])
+                             configs=['dbg', 'opt'],
+                             platforms=['linux', 'macos', 'windows'],
+                             labels=['basictests'],
+                             extra_args=extra_args)
   
   # supported on linux and mac.
   test_jobs += _generate_jobs(languages=['c++', 'ruby', 'php'],
-                             configs=['dbg', 'opt'],
-                             platforms=['linux', 'macos'],
-                             labels=['basictests'])
+                              configs=['dbg', 'opt'],
+                              platforms=['linux', 'macos'],
+                              labels=['basictests'],
+                              extra_args=extra_args)
   
   # supported on mac only.
   test_jobs += _generate_jobs(languages=['objc'],
                               configs=['dbg', 'opt'],
                               platforms=['macos'],
-                              labels=['basictests'])
+                              labels=['basictests'],
+                              extra_args=extra_args)
   
   # sanitizers
   test_jobs += _generate_jobs(languages=['c'],
                               configs=['msan', 'asan', 'tsan'],
                               platforms=['linux'],
-                              labels=['sanitizers'])
+                              labels=['sanitizers'],
+                              extra_args=extra_args)
   test_jobs += _generate_jobs(languages=['c++'],
                               configs=['asan', 'tsan'],
                               platforms=['linux'],
-                              labels=['sanitizers'])
+                              labels=['sanitizers'],
+                              extra_args=extra_args)
   return test_jobs
 
   
-def _create_portability_test_jobs():
+def _create_portability_test_jobs(extra_args=[]):
   test_jobs = []
   # portability C x86
   test_jobs += _generate_jobs(languages=['c'],
@@ -148,7 +155,8 @@ def _create_portability_test_jobs():
                               platforms=['linux'],
                               arch='x86',
                               compiler='default',
-                              labels=['portability'])
+                              labels=['portability'],
+                              extra_args=extra_args)
   
   # portability C and C++ on x64
   for compiler in ['gcc4.4', 'gcc4.6', 'gcc5.3',
@@ -158,7 +166,8 @@ def _create_portability_test_jobs():
                                 platforms=['linux'],
                                 arch='x64',
                                 compiler=compiler,
-                                labels=['portability'])
+                                labels=['portability'],
+                                extra_args=extra_args)
   
   # portability C on Windows
   for arch in ['x86', 'x64']:
@@ -168,53 +177,72 @@ def _create_portability_test_jobs():
                                   platforms=['windows'],
                                   arch=arch,
                                   compiler=compiler,
-                                  labels=['portability'])
+                                  labels=['portability'],
+                                  extra_args=extra_args)
   
   test_jobs += _generate_jobs(languages=['python'],
                               configs=['dbg'],
                               platforms=['linux'],
                               arch='default',
                               compiler='python3.4',
-                              labels=['portability'])
+                              labels=['portability'],
+                              extra_args=extra_args)
   
   test_jobs += _generate_jobs(languages=['csharp'],
                               configs=['dbg'],
                               platforms=['linux'],
                               arch='default',
                               compiler='coreclr',
-                              labels=['portability'])
+                              labels=['portability'],
+                              extra_args=extra_args)
   
   for compiler in ['node5', 'node6', 'node0.12']:
     test_jobs += _generate_jobs(languages=['node'],
-                               configs=['dbg'],
-                               platforms=['linux'],
-                               arch='default',
-                               compiler=compiler,
-                               labels=['portability'])
+                                configs=['dbg'],
+                                platforms=['linux'],
+                                arch='default',
+                                compiler=compiler,
+                                labels=['portability'],
+                                extra_args=extra_args)
   return test_jobs  
 
 
-all_jobs = _create_test_jobs() + _create_portability_test_jobs()
+def _allowed_labels():
+  """Returns a list of existing job labels."""
+  all_labels = set()
+  for job in _create_test_jobs() + _create_portability_test_jobs():
+    for label in job.labels:
+      all_labels.add(label)
+  return sorted(all_labels)
 
-all_labels = set()
-for job in all_jobs:
-  for label in job.labels:
-    all_labels.add(label)
 
 argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
-# TODO(jtattermusch): allow running tests with --build_only flag
-# TODO(jtattermusch): allow running tests with --force_default_poller flag.
 argp.add_argument('-j', '--jobs',
                   default=multiprocessing.cpu_count()/_INNER_JOBS,
                   type=int,
                   help='Number of concurrent run_tests.py instances.')
 argp.add_argument('-f', '--filter',
-                  choices=sorted(all_labels),
+                  choices=_allowed_labels(),
                   nargs='+',
                   default=[],
                   help='Filter targets to run by label with AND semantics.')
+argp.add_argument('--build_only',
+                  default=False,
+                  action='store_const',
+                  const=True,
+                  help='Pass --build_only flag to run_tests.py instances.')
+argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
+                  help='Pass --force_default_poller to run_tests.py instances.')
 args = argp.parse_args()
 
+extra_args = []
+if args.build_only:
+  extra_args.append('--build_only')
+if args.force_default_poller:
+  extra_args.append('--force_default_poller')
+
+all_jobs = _create_test_jobs(extra_args=extra_args) + _create_portability_test_jobs(extra_args=extra_args)
+
 jobs = []
 for job in all_jobs:
   if not args.filter or all(filter in job.labels for filter in args.filter):

From f098c753baf059b5c01699b8f56c194180cbf3e1 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 22 Sep 2016 14:28:07 +0200
Subject: [PATCH 29/52] node6 portability test not yet implemented

---
 tools/run_tests/run_tests_matrix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index ae7fdd84f9b..d87632d8891 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -196,7 +196,7 @@ def _create_portability_test_jobs(extra_args=[]):
                               labels=['portability'],
                               extra_args=extra_args)
   
-  for compiler in ['node5', 'node6', 'node0.12']:
+  for compiler in ['node5', 'node0.12']:
     test_jobs += _generate_jobs(languages=['node'],
                                 configs=['dbg'],
                                 platforms=['linux'],

From e922d7702eb8ead6acfa23fc44823b699bc6a72f Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 22 Sep 2016 14:31:10 +0200
Subject: [PATCH 30/52] support --dry_run flag

---
 tools/run_tests/run_tests_matrix.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index d87632d8891..a31b0dd9177 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -233,6 +233,11 @@ argp.add_argument('--build_only',
                   help='Pass --build_only flag to run_tests.py instances.')
 argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
                   help='Pass --force_default_poller to run_tests.py instances.')
+argp.add_argument('--dry_run',
+                  default=False,
+                  action='store_const',
+                  const=True,
+                  help='Only print what would be run.')
 args = argp.parse_args()
 
 extra_args = []
@@ -263,6 +268,10 @@ for job in jobs:
   print '  %s' % job.shortname
 print
 
+if args.dry_run:
+  print '--dry_run was used, exiting'
+  sys.exit(1)
+
 jobset.message('START', 'Running test matrix.', do_newline=True)
 num_failures, resultset = jobset.run(jobs,
                                      newline_on_success=True,

From b6b4f8759b641aee0d737d16d8375560d9024e4d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Thu, 22 Sep 2016 14:44:27 +0200
Subject: [PATCH 31/52] better --dry_run printouts

---
 tools/run_tests/run_tests_matrix.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index a31b0dd9177..d5f90478257 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -265,7 +265,10 @@ print('copied to the docker environment or into subworkspaces.')
 print 
 print 'Will run these tests:'
 for job in jobs:
-  print '  %s' % job.shortname
+  if args.dry_run:
+    print '  %s: "%s"' % (job.shortname, ' '.join(job.cmdline))
+  else:
+    print '  %s' % job.shortname
 print
 
 if args.dry_run:

From 1b0dd825cc6e56fa79c5de46452069aee87193e5 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 4 Oct 2016 11:52:32 +0200
Subject: [PATCH 32/52] fixes for run_tests_in_workspace.sh

---
 tools/run_tests/run_tests_in_workspace.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/run_tests_in_workspace.sh b/tools/run_tests/run_tests_in_workspace.sh
index b0c9ad05f76..98ef3566db1 100755
--- a/tools/run_tests/run_tests_in_workspace.sh
+++ b/tools/run_tests/run_tests_in_workspace.sh
@@ -42,5 +42,5 @@ rm -rf "${WORKSPACE_NAME}"
 git clone --recursive . "${WORKSPACE_NAME}"
 
 echo "Running run_tests.py in workspace ${WORKSPACE_NAME}" 
-"${WORKSPACE_NAME}/tools/run_tests/run_tests.py" $@
+python "${WORKSPACE_NAME}/tools/run_tests/run_tests.py" $@
 

From ad666a94142db43593456f4cd88645ff80e66786 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 4 Oct 2016 15:08:36 +0200
Subject: [PATCH 33/52] remove node portability targets

---
 tools/run_tests/run_tests_matrix.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index d5f90478257..a94f9cfef5e 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -195,15 +195,6 @@ def _create_portability_test_jobs(extra_args=[]):
                               compiler='coreclr',
                               labels=['portability'],
                               extra_args=extra_args)
-  
-  for compiler in ['node5', 'node0.12']:
-    test_jobs += _generate_jobs(languages=['node'],
-                                configs=['dbg'],
-                                platforms=['linux'],
-                                arch='default',
-                                compiler=compiler,
-                                labels=['portability'],
-                                extra_args=extra_args)
   return test_jobs  
 
 

From ffdd3a3cab09235c11efc34bea2ee6746dd54f12 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Tue, 4 Oct 2016 17:52:15 +0200
Subject: [PATCH 34/52] add proxy script for run_tests_matrix.py

---
 tools/jenkins/run_jenkins_matrix.sh | 42 +++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100755 tools/jenkins/run_jenkins_matrix.sh

diff --git a/tools/jenkins/run_jenkins_matrix.sh b/tools/jenkins/run_jenkins_matrix.sh
new file mode 100755
index 00000000000..b3783e69584
--- /dev/null
+++ b/tools/jenkins/run_jenkins_matrix.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This script is invoked by Jenkins and triggers a test run, bypassing
+# all args to the test script.
+#
+# Setting up rvm environment BEFORE we set -ex.
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+# To prevent cygwin bash complaining about empty lines ending with \r
+# we set the igncr option. The option doesn't exist on Linux, so we fallback
+# to just 'set -ex' there.
+# NOTE: No empty lines should appear in this file before igncr is set!
+set -ex -o igncr || set -ex
+
+python tools/run_tests/run_tests_matrix.py $@

From 5f33e0b1d4c6c6ec394967afba703fdb76c73738 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Mon, 3 Oct 2016 19:47:37 +0200
Subject: [PATCH 35/52] fix building go interop image

---
 .../interoptest/grpc_interop_go/build_interop.sh    | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh
index 1fd088322cd..858ee0a68cd 100755
--- a/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh
+++ b/tools/dockerfile/interoptest/grpc_interop_go/build_interop.sh
@@ -36,19 +36,12 @@ set -e
 # to test instead of using "go get" to download from Github directly.
 git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc
 
+# Get all gRPC Go dependencies
+(cd src/google.golang.org/grpc && go get -t .)
+
 # copy service account keys if available
 cp -r /var/local/jenkins/service_account $HOME || true
 
-# Get dependencies from GitHub
-# NOTE: once grpc-go dependencies change, this needs to be updated manually
-# but we don't expect this to happen any time soon.
-go get github.com/golang/protobuf/proto
-go get golang.org/x/net/context
-go get golang.org/x/net/trace
-go get golang.org/x/oauth2
-go get golang.org/x/oauth2/google
-go get google.golang.org/cloud
-
 # Build the interop client and server
 (cd src/google.golang.org/grpc/interop/client && go install)
 (cd src/google.golang.org/grpc/interop/server && go install)

From 82b64d1565318afb40d72d46e890766d24368dc8 Mon Sep 17 00:00:00 2001
From: Alexander Polcyn <apolcyn@google.com>
Date: Thu, 6 Oct 2016 17:33:12 -0700
Subject: [PATCH 36/52] change back slashes to forward slashes in grpc.tool
 nuspec

---
 src/csharp/Grpc.Tools.nuspec | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/csharp/Grpc.Tools.nuspec b/src/csharp/Grpc.Tools.nuspec
index 0c937ab9cbf..ba4e1d674cd 100644
--- a/src/csharp/Grpc.Tools.nuspec
+++ b/src/csharp/Grpc.Tools.nuspec
@@ -17,17 +17,17 @@
   </metadata>
   <files>
     <!-- forward slashes in src path enable building on Linux -->
-    <file src="protoc_plugins/windows_x86/protoc.exe" target="tools\windows_x86\protoc.exe" />
-    <file src="protoc_plugins/windows_x86/grpc_csharp_plugin.exe" target="tools\windows_x86\grpc_csharp_plugin.exe" />
-    <file src="protoc_plugins/windows_x64/protoc.exe" target="tools\windows_x64\protoc.exe" />
-    <file src="protoc_plugins/windows_x64/grpc_csharp_plugin.exe" target="tools\windows_x64\grpc_csharp_plugin.exe" />
-    <file src="protoc_plugins/linux_x86/protoc" target="tools\linux_x86\protoc" />
-    <file src="protoc_plugins/linux_x86/grpc_csharp_plugin" target="tools\linux_x86\grpc_csharp_plugin" />
-    <file src="protoc_plugins/linux_x64/protoc" target="tools\linux_x64\protoc" />
-    <file src="protoc_plugins/linux_x64/grpc_csharp_plugin" target="tools\linux_x64\grpc_csharp_plugin" />
-    <file src="protoc_plugins/macosx_x86/protoc" target="tools\macosx_x86\protoc" />
-    <file src="protoc_plugins/macosx_x86/grpc_csharp_plugin" target="tools\macosx_x86\grpc_csharp_plugin" />
-    <file src="protoc_plugins/macosx_x64/protoc" target="tools\macosx_x64\protoc" />
-    <file src="protoc_plugins/macosx_x64/grpc_csharp_plugin" target="tools\macosx_x64\grpc_csharp_plugin" />
+    <file src="protoc_plugins/windows_x86/protoc.exe" target="tools/windows_x86/protoc.exe" />
+    <file src="protoc_plugins/windows_x86/grpc_csharp_plugin.exe" target="tools/windows_x86/grpc_csharp_plugin.exe" />
+    <file src="protoc_plugins/windows_x64/protoc.exe" target="tools/windows_x64/protoc.exe" />
+    <file src="protoc_plugins/windows_x64/grpc_csharp_plugin.exe" target="tools/windows_x64/grpc_csharp_plugin.exe" />
+    <file src="protoc_plugins/linux_x86/protoc" target="tools/linux_x86/protoc" />
+    <file src="protoc_plugins/linux_x86/grpc_csharp_plugin" target="tools/linux_x86/grpc_csharp_plugin" />
+    <file src="protoc_plugins/linux_x64/protoc" target="tools/linux_x64/protoc" />
+    <file src="protoc_plugins/linux_x64/grpc_csharp_plugin" target="tools/linux_x64/grpc_csharp_plugin" />
+    <file src="protoc_plugins/macosx_x86/protoc" target="tools/macosx_x86/protoc" />
+    <file src="protoc_plugins/macosx_x86/grpc_csharp_plugin" target="tools/macosx_x86/grpc_csharp_plugin" />
+    <file src="protoc_plugins/macosx_x64/protoc" target="tools/macosx_x64/protoc" />
+    <file src="protoc_plugins/macosx_x64/grpc_csharp_plugin" target="tools/macosx_x64/grpc_csharp_plugin" />
   </files>
 </package>

From ab7abdb968d0a455e3b0ed2f33722243283e9bb9 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 7 Oct 2016 12:59:08 +0200
Subject: [PATCH 37/52] dont eat run_tests.py errors on test failure

---
 tools/run_tests/dockerize/build_docker_and_run_tests.sh | 8 +++-----
 tools/run_tests/run_tests.py                            | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index b4b172ddef6..c3219c533d5 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -61,6 +61,7 @@ CONTAINER_NAME="run_tests_$(uuidgen)"
 docker_instance_git_root=/var/local/jenkins/grpc
 
 # Run tests inside docker
+DOCKER_EXIT_CODE=0
 docker run \
   -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND" \
   -e "config=$config" \
@@ -81,7 +82,7 @@ docker run \
   -w /var/local/git/grpc \
   --name=$CONTAINER_NAME \
   $DOCKER_IMAGE_NAME \
-  bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_FAILED="true"
+  bash -l "/var/local/jenkins/grpc/$DOCKER_RUN_SCRIPT" || DOCKER_EXIT_CODE=$?
 
 # use unique name for reports.zip to prevent clash between concurrent
 # run_tests.py runs 
@@ -93,7 +94,4 @@ rm -f ${TEMP_REPORTS_ZIP}
 # remove the container, possibly killing it first
 docker rm -f $CONTAINER_NAME || true
 
-if [ "$DOCKER_FAILED" != "" ] && [ "$XML_REPORT" == "" ]
-then
-  exit 1
-fi
+exit $DOCKER_EXIT_CODE
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 3ccba877c99..c7d10e057f5 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -1369,7 +1369,7 @@ else:
   exit_code = 0
   if BuildAndRunError.BUILD in errors:
     exit_code |= 1
-  if BuildAndRunError.TEST in errors and not args.travis:
+  if BuildAndRunError.TEST in errors:
     exit_code |= 2
   if BuildAndRunError.POST_TEST in errors:
     exit_code |= 4

From 2b39808e1c998f67b32d3c2bda0b7bb64a92db76 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@google.com>
Date: Fri, 7 Oct 2016 14:40:30 +0200
Subject: [PATCH 38/52] fix building ruby artifact

---
 src/core/ext/lb_policy/grpclb/grpclb.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c
index 63af774ea6d..a64414877f0 100644
--- a/src/core/ext/lb_policy/grpclb/grpclb.c
+++ b/src/core/ext/lb_policy/grpclb/grpclb.c
@@ -329,8 +329,8 @@ static bool is_server_valid(const grpc_grpclb_server *server, size_t idx,
   if (server->port >> 16 != 0) {
     if (log) {
       gpr_log(GPR_ERROR,
-              "Invalid port '%d' at index %zu of serverlist. Ignoring.",
-              server->port, idx);
+              "Invalid port '%d' at index %lu of serverlist. Ignoring.",
+              server->port, (unsigned long)idx);
     }
     return false;
   }
@@ -338,9 +338,9 @@ static bool is_server_valid(const grpc_grpclb_server *server, size_t idx,
   if (ip->size != 4 && ip->size != 16) {
     if (log) {
       gpr_log(GPR_ERROR,
-              "Expected IP to be 4 or 16 bytes, got %d at index %zu of "
+              "Expected IP to be 4 or 16 bytes, got %d at index %lu of "
               "serverlist. Ignoring",
-              ip->size, idx);
+              ip->size, (unsigned long)idx);
     }
     return false;
   }
@@ -1070,8 +1070,8 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
     if (serverlist != NULL) {
       gpr_slice_unref(response_slice);
       if (grpc_lb_glb_trace) {
-        gpr_log(GPR_INFO, "Serverlist with %zu servers received",
-                serverlist->num_servers);
+        gpr_log(GPR_INFO, "Serverlist with %lu servers received",
+                (unsigned long)serverlist->num_servers);
       }
 
       /* update serverlist */
@@ -1155,10 +1155,10 @@ static void srv_status_rcvd_cb(grpc_exec_ctx *exec_ctx, void *arg,
   if (grpc_lb_glb_trace) {
     gpr_log(GPR_INFO,
             "status from lb server received. Status = %d, Details = '%s', "
-            "Capaticy "
-            "= %zu",
+            "Capacity "
+            "= %lu",
             lb_client->status, lb_client->status_details,
-            lb_client->status_details_capacity);
+            (unsigned long)lb_client->status_details_capacity);
   }
   /* TODO(dgq): deal with stream termination properly (fire up another one?
    * fail the original call?) */

From 517503746c65944e9783b00ef004c403b6bd18a9 Mon Sep 17 00:00:00 2001
From: Robbie Shade <rjshade@google.com>
Date: Mon, 10 Oct 2016 14:49:55 -0400
Subject: [PATCH 39/52] Check for the correct number of orphan callbacks in
 udp_server_test

---
 test/core/iomgr/udp_server_test.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c
index 2a304275043..71d2fb5bd44 100644
--- a/test/core/iomgr/udp_server_test.c
+++ b/test/core/iomgr/udp_server_test.c
@@ -131,8 +131,9 @@ static void test_no_op_with_port_and_start(void) {
   grpc_udp_server_destroy(&exec_ctx, s, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 
-  /* The server had a single FD, which should have been orphaned. */
-  GPR_ASSERT(g_number_of_orphan_calls == 1);
+  /* The server had a single FD, which is orphaned once in *
+   * deactivated_all_ports, and once in grpc_udp_server_destroy. */
+  GPR_ASSERT(g_number_of_orphan_calls == 2);
 }
 
 static void test_receive(int number_of_clients) {
@@ -196,8 +197,9 @@ static void test_receive(int number_of_clients) {
   grpc_udp_server_destroy(&exec_ctx, s, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 
-  /* The server had a single FD, which should have been orphaned. */
-  GPR_ASSERT(g_number_of_orphan_calls == 1);
+  /* The server had a single FD, which is orphaned once in *
+   * deactivated_all_ports, and once in grpc_udp_server_destroy. */
+  GPR_ASSERT(g_number_of_orphan_calls == 2);
 }
 
 static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p,

From 41c06a2ed563e31add33212dd861f554fc426c2b Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Tue, 11 Oct 2016 13:49:23 -0700
Subject: [PATCH 40/52] move changes from dockerfile to template

---
 .../test/cxx_wheezy_x64/Dockerfile.template          |  6 ++++++
 tools/dockerfile/test/cxx_wheezy_x64/Dockerfile      | 12 +++++-------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template
index b6a3b0d5d21..5c38e9a0e92 100644
--- a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template
+++ b/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template
@@ -41,6 +41,12 @@
     g++-4.4 ${'\\'}
     g++-4.4-multilib
 
+  # set up backport to allow installation of Git version > 1.7
+  RUN echo "deb http://http.debian.net/debian wheezy-backports main" \
+    >/etc/apt/sources.list.d/wheezy-backports.list
+  RUN apt-get update -qq
+  RUN apt-get -t wheezy-backports install -qq git
+  
   RUN wget ${openssl_fallback.base_uri + openssl_fallback.tarball}
 
   ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh
diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
index 503a4357070..f22fcacc139 100644
--- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
@@ -63,13 +63,6 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
-#================
-# Add backport to Debian sources.list and update git to version > 1.7
-RUN echo "deb http://http.debian.net/debian wheezy-backports main" \
-  >/etc/apt/sources.list.d/wheezy-backports.list
-RUN apt-get update -qq
-RUN apt-get -t wheezy-backports install -y -qq git mercurial
-
 #====================
 # Python dependencies
 
@@ -96,6 +89,11 @@ RUN apt-get update && apt-get install -y \
   g++-4.4 \
   g++-4.4-multilib
 
+# set up backport to allow installation of Git version > 1.7
+RUN echo "deb http://http.debian.net/debian wheezy-backports main"   >/etc/apt/sources.list.d/wheezy-backports.list
+RUN apt-get update -qq
+RUN apt-get -t wheezy-backports install -qq git
+
 RUN wget https://openssl.org/source/old/1.0.2/openssl-1.0.2f.tar.gz
 
 ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh

From 580e976b5fb23cc394d710a511d896f75fc9ce31 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 17:42:55 +0200
Subject: [PATCH 41/52] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e9b0454871c..70cf9fcc30e 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ See [tools/run_tests](tools/run_tests) for more guidance on how to run various t
 
 This repository contains source code for gRPC libraries for multiple languages written on top of shared C core library [src/core] (src/core).
 
-Libraries in different languages are in different states of development. We are seeking contributions for all of these libraries.
+Libraries in different languages may be in different states of development. We are seeking contributions for all of these libraries.
 
 | Language                | Source                              | Status  |
 |-------------------------|-------------------------------------|---------|

From 10edbe5485add0cb1b1e4b71045d783f1a1ed3b8 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 17:48:59 +0200
Subject: [PATCH 42/52] Update README.md

---
 src/core/README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/core/README.md b/src/core/README.md
index 0d8c0d5bd90..62865d0c13e 100644
--- a/src/core/README.md
+++ b/src/core/README.md
@@ -2,7 +2,3 @@
 
 This directory contains source code for shared C library. Libraries in other languages in this repository (C++, Ruby,
 Python, PHP, NodeJS, Objective-C) are layered on top of this library.
-
-#Status
-
-Beta

From 7a82301a2ea55fd534bb0e43c68c0d7e94878dff Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 17:55:57 +0200
Subject: [PATCH 43/52] Update README.md

---
 src/core/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/core/README.md b/src/core/README.md
index 62865d0c13e..44c6f247725 100644
--- a/src/core/README.md
+++ b/src/core/README.md
@@ -1,4 +1,4 @@
 #Overview
 
-This directory contains source code for shared C library. Libraries in other languages in this repository (C++, Ruby,
+This directory contains source code for C library (a.k.a the *gRPC C core*) that provides all gRPC's core functionality through a low level API. Libraries in other languages in this repository (C++, Ruby,
 Python, PHP, NodeJS, Objective-C) are layered on top of this library.

From 5b2e172e5785bcd73c425192ca3d670a96044d27 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:00:39 +0200
Subject: [PATCH 44/52] Create README.md

---
 src/compiler/README.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 src/compiler/README.md

diff --git a/src/compiler/README.md b/src/compiler/README.md
new file mode 100644
index 00000000000..a2f49b3cd53
--- /dev/null
+++ b/src/compiler/README.md
@@ -0,0 +1,4 @@
+#Overview
+
+This directory contains source code for gRPC protocol buffer compiler (*protoc*) plugins. Along with `protoc`,
+these plugins are used to generate gRPC client and server stubs from `.proto` files.

From 9cece7ca992c6594ed163dc9996a3f879f3d7a1f Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:01:41 +0200
Subject: [PATCH 45/52] Update README.md

---
 src/cpp/README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/cpp/README.md b/src/cpp/README.md
index 8c0f85e5ff7..d9b521317a6 100644
--- a/src/cpp/README.md
+++ b/src/cpp/README.md
@@ -3,10 +3,6 @@
 
 This directory contains source code for C++ implementation of gRPC.
 
-#Status
-
-Beta
-
 #Pre-requisites
 
 ##Linux

From 9633bd356f2fd90c1bccf0be58f406316155ca54 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:02:32 +0200
Subject: [PATCH 46/52] Update README.md

---
 src/csharp/README.md | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/csharp/README.md b/src/csharp/README.md
index 18d5945a8a1..a04172dad4f 100644
--- a/src/csharp/README.md
+++ b/src/csharp/README.md
@@ -4,11 +4,6 @@ gRPC C#
 
 A C# implementation of gRPC.
 
-Status
-------
-
-Beta
-
 PREREQUISITES
 --------------
 

From d1395d9a667b43807b6f068eff06f4fb53cf43d0 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:03:00 +0200
Subject: [PATCH 47/52] Update README.md

---
 src/node/README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/node/README.md b/src/node/README.md
index 15d4c6d02f7..fc407435cd7 100644
--- a/src/node/README.md
+++ b/src/node/README.md
@@ -1,9 +1,6 @@
 [![npm](https://img.shields.io/npm/v/grpc.svg)](https://www.npmjs.com/package/grpc)
 # Node.js gRPC Library
 
-## Status
-Beta
-
 ## PREREQUISITES
 - `node`: This requires `node` to be installed, version `0.12` or above. If you instead have the `nodejs` executable on Debian, you should install the [`nodejs-legacy`](https://packages.debian.org/sid/nodejs-legacy) package.
 

From 45775fd8a13a4a92310739672d6eba29a8e8e0e6 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:03:47 +0200
Subject: [PATCH 48/52] Update README.md

---
 src/php/README.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/php/README.md b/src/php/README.md
index 7e9819b256d..f2ad96f7a89 100644
--- a/src/php/README.md
+++ b/src/php/README.md
@@ -3,10 +3,6 @@
 
 This directory contains source code for PHP implementation of gRPC layered on shared C library.
 
-#Status
-
-GA
-
 ## Environment
 
 Prerequisite:

From 5d77c964fc881df3f8c2f2cff274b46134d48943 Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:04:37 +0200
Subject: [PATCH 49/52] Update README.md

---
 src/ruby/README.md | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/ruby/README.md b/src/ruby/README.md
index 31795754866..a4d7b56b4ec 100644
--- a/src/ruby/README.md
+++ b/src/ruby/README.md
@@ -4,11 +4,6 @@ gRPC Ruby
 
 A Ruby implementation of gRPC.
 
-Status
-------
-
-Beta
-
 PREREQUISITES
 -------------
 

From 0dc8d354e9546bb15fd5b3342637ed8b1d1cd10d Mon Sep 17 00:00:00 2001
From: Jan Tattermusch <jtattermusch@users.noreply.github.com>
Date: Wed, 12 Oct 2016 18:22:39 +0200
Subject: [PATCH 50/52] Update README.md

---
 src/csharp/README.md | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/csharp/README.md b/src/csharp/README.md
index a04172dad4f..0405ff88a01 100644
--- a/src/csharp/README.md
+++ b/src/csharp/README.md
@@ -4,6 +4,14 @@ gRPC C#
 
 A C# implementation of gRPC.
 
+SUPPORTED PLATFORMS
+------------------
+
+- .NET Framework 4.5+ (Windows)
+- [.NET Core](https://dotnet.github.io/) on Linux, Windows and Mac OS X (starting from version 1.0.1)
+- Mono 4+ on Linux, Windows and Mac OS X
+
+
 PREREQUISITES
 --------------
 
@@ -11,6 +19,7 @@ PREREQUISITES
 - Linux: Mono 4+, MonoDevelop 5.9+ (with NuGet add-in installed)
 - Mac OS X: Xamarin Studio 5.9+
 
+
 HOW TO USE
 --------------
 
@@ -64,12 +73,6 @@ different languages.
 tools/run_tests/run_tests.py -l csharp
 ```
 
-ON .NET CORE SUPPORT
-------------------
-
-We are committed to providing full support for [.NET Core](https://dotnet.github.io/) in near future,
-but currently, the support is for .NET Core is experimental/work-in-progress.
-
 DOCUMENTATION
 -------------
 - [API Reference][]
@@ -97,9 +100,7 @@ CONTENTS
 THE NATIVE DEPENDENCY
 ---------------
 
-Internally, gRPC C# uses a native library written in C (gRPC C core) and invokes its functionality via P/Invoke. `grpc_csharp_ext` library is a native extension library that facilitates this by wrapping some C core API into a form that's more digestible for P/Invoke.
-
-Prior to version 0.13, installing `grpc_csharp_ext` was required to make gRPC work on Linux and MacOS. Starting with version 0.13, we have improved the packaging story significantly and precompiled versions of the native library for all supported platforms are now shipped with the NuGet package. Just installing the `Grpc` NuGet package should be the only step needed to use gRPC C#, regardless of your platform (Windows, Linux or Mac) and the bitness (32 or 64bit).
+Internally, gRPC C# uses a native library written in C (gRPC C core) and invokes its functionality via P/Invoke. The fact that a native library is used should be fully transparent to the users and just installing the `Grpc.Core` NuGet package is the only step needed to use gRPC C# on all supported platforms.
 
 [API Reference]: http://www.grpc.io/grpc/csharp/
 [Helloworld Example]: ../../examples/csharp/helloworld

From 82d7677ec9104e6e3bf57097591ef322758e6cfb Mon Sep 17 00:00:00 2001
From: Makarand Dharmapurikar <makarandd@google.com>
Date: Wed, 12 Oct 2016 10:28:31 -0700
Subject: [PATCH 51/52] trivial change to kick off jenkins run

---
 doc/interop-test-descriptions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md
index 97d76191a8f..666af185b9f 100644
--- a/doc/interop-test-descriptions.md
+++ b/doc/interop-test-descriptions.md
@@ -64,7 +64,7 @@ ensure that the proto serialized to zero bytes.*
 
 This test verifies that gRPC requests marked as cacheable use GET verb instead
 of POST, and that server sets appropriate cache control headers for the response
-to be cached by a proxy. This interop test requires that the server is behind
+to be cached by a proxy. This test requires that the server is behind
 a caching proxy. Use of current timestamp in the request prevents accidental
 cache matches left over from previous tests.
 

From b5299976c14781bc98890a0c2d4b6513804c264e Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Wed, 12 Oct 2016 14:13:42 -0700
Subject: [PATCH 52/52] Fixed clang-format sanity error

---
 test/cpp/interop/interop_server.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index 05af18bc671..58f20aa611a 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -155,8 +155,9 @@ class TestServiceImpl : public TestService::Service {
   }
 
   // Response contains current timestamp. We ignore everything in the request.
-  Status CacheableUnaryCall(ServerContext* context, const SimpleRequest* request,
-                   SimpleResponse* response) {
+  Status CacheableUnaryCall(ServerContext* context,
+                            const SimpleRequest* request,
+                            SimpleResponse* response) {
     gpr_timespec ts = gpr_now(GPR_CLOCK_PRECISE);
     std::string timestamp = std::to_string((long long unsigned)ts.tv_nsec);
     response->mutable_payload()->set_body(timestamp.c_str(), timestamp.size());