From a651bf1b26c5e94d482ad98a4974f78695c25f99 Mon Sep 17 00:00:00 2001
From: Matt Kwong <mattkwong@google.com>
Date: Fri, 10 Feb 2017 15:31:02 -0800
Subject: [PATCH 01/10] Increase test timeout duration for internal CI

---
 tools/internal_ci/linux/grpc_master.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/internal_ci/linux/grpc_master.cfg b/tools/internal_ci/linux/grpc_master.cfg
index 8ce2ef11a20..d5901bb6010 100644
--- a/tools/internal_ci/linux/grpc_master.cfg
+++ b/tools/internal_ci/linux/grpc_master.cfg
@@ -32,7 +32,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_master.sh"
-timeout_mins: 60
+timeout_mins: 240
 action {
   define_artifacts {
     regex: "**/sponge_log.xml"

From 14ae5381af80f5a1fc30e6e88f94cfbe314d54f8 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Tue, 14 Feb 2017 12:43:50 -0800
Subject: [PATCH 02/10] Add microbenchmark of single-threaded CQ operations

---
 CMakeLists.txt                                |  41 ++++++
 Makefile                                      |  48 +++++++
 build.yaml                                    |  19 +++
 test/cpp/microbenchmarks/bm_cq.cc             | 118 ++++++++++++++++++
 .../generated/sources_and_headers.json        |  20 +++
 tools/run_tests/generated/tests.json          |  22 ++++
 6 files changed, 268 insertions(+)
 create mode 100644 test/cpp/microbenchmarks/bm_cq.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 101f0ad2ef9..bf267492218 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -570,6 +570,9 @@ add_dependencies(buildtests_cxx alarm_cpp_test)
 add_dependencies(buildtests_cxx async_end2end_test)
 add_dependencies(buildtests_cxx auth_property_iterator_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+add_dependencies(buildtests_cxx bm_cq)
+endif()
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx bm_fullstack)
 endif()
 add_dependencies(buildtests_cxx channel_arguments_test)
@@ -7445,6 +7448,44 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 
+add_executable(bm_cq
+  test/cpp/microbenchmarks/bm_cq.cc
+  third_party/googletest/src/gtest-all.cc
+)
+
+
+target_include_directories(bm_cq
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${BORINGSSL_ROOT_DIR}/include
+  PRIVATE ${PROTOBUF_ROOT_DIR}/src
+  PRIVATE ${BENCHMARK_ROOT_DIR}/include
+  PRIVATE ${ZLIB_ROOT_DIR}
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
+  PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
+  PRIVATE third_party/googletest/include
+  PRIVATE third_party/googletest
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(bm_cq
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  benchmark
+  grpc++_test_util
+  grpc_test_util
+  grpc++
+  grpc
+  gpr_test_util
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+endif()
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+
 add_executable(bm_fullstack
   test/cpp/microbenchmarks/bm_fullstack.cc
   third_party/googletest/src/gtest-all.cc
diff --git a/Makefile b/Makefile
index 5dde577cca8..313dce05325 100644
--- a/Makefile
+++ b/Makefile
@@ -1043,6 +1043,7 @@ wakeup_fd_cv_test: $(BINDIR)/$(CONFIG)/wakeup_fd_cv_test
 alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test
 async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test
 auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test
+bm_cq: $(BINDIR)/$(CONFIG)/bm_cq
 bm_fullstack: $(BINDIR)/$(CONFIG)/bm_fullstack
 channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test
 channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test
@@ -1449,6 +1450,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/alarm_cpp_test \
   $(BINDIR)/$(CONFIG)/async_end2end_test \
   $(BINDIR)/$(CONFIG)/auth_property_iterator_test \
+  $(BINDIR)/$(CONFIG)/bm_cq \
   $(BINDIR)/$(CONFIG)/bm_fullstack \
   $(BINDIR)/$(CONFIG)/channel_arguments_test \
   $(BINDIR)/$(CONFIG)/channel_filter_test \
@@ -1553,6 +1555,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/alarm_cpp_test \
   $(BINDIR)/$(CONFIG)/async_end2end_test \
   $(BINDIR)/$(CONFIG)/auth_property_iterator_test \
+  $(BINDIR)/$(CONFIG)/bm_cq \
   $(BINDIR)/$(CONFIG)/bm_fullstack \
   $(BINDIR)/$(CONFIG)/channel_arguments_test \
   $(BINDIR)/$(CONFIG)/channel_filter_test \
@@ -1870,6 +1873,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/async_end2end_test || ( echo test async_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing auth_property_iterator_test"
 	$(Q) $(BINDIR)/$(CONFIG)/auth_property_iterator_test || ( echo test auth_property_iterator_test failed ; exit 1 )
+	$(E) "[RUN]     Testing bm_cq"
+	$(Q) $(BINDIR)/$(CONFIG)/bm_cq || ( echo test bm_cq failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_fullstack"
 	$(Q) $(BINDIR)/$(CONFIG)/bm_fullstack || ( echo test bm_fullstack failed ; exit 1 )
 	$(E) "[RUN]     Testing channel_arguments_test"
@@ -12389,6 +12394,49 @@ endif
 endif
 
 
+BM_CQ_SRC = \
+    test/cpp/microbenchmarks/bm_cq.cc \
+
+BM_CQ_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_CQ_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/bm_cq: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/bm_cq: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/bm_cq: $(PROTOBUF_DEP) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o:  $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_bm_cq: $(BM_CQ_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(BM_CQ_OBJS:.o=.dep)
+endif
+endif
+
+
 BM_FULLSTACK_SRC = \
     test/cpp/microbenchmarks/bm_fullstack.cc \
 
diff --git a/build.yaml b/build.yaml
index 2fbaa2e2262..293d7584787 100644
--- a/build.yaml
+++ b/build.yaml
@@ -2984,6 +2984,25 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
+- name: bm_cq
+  build: test
+  language: c++
+  src:
+  - test/cpp/microbenchmarks/bm_cq.cc
+  deps:
+  - benchmark
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr_test_util
+  - gpr
+  args:
+  - --benchmark_min_time=0
+  platforms:
+  - mac
+  - linux
+  - posix
 - name: bm_fullstack
   build: test
   language: c++
diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc
new file mode 100644
index 00000000000..502320d6dc3
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_cq.cc
@@ -0,0 +1,118 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* This benchmark exists to ensure that the benchmark integration is
+ * working */
+
+#include <grpc++/completion_queue.h>
+#include <grpc++/impl/grpc_library.h>
+#include <grpc/grpc.h>
+
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+extern "C" {
+#include "src/core/lib/surface/completion_queue.h"
+}
+
+namespace grpc {
+namespace testing {
+
+static class InitializeStuff {
+ public:
+  InitializeStuff() { init_lib_.init(); }
+  ~InitializeStuff() { init_lib_.shutdown(); }
+
+ private:
+  internal::GrpcLibrary init_lib_;
+  internal::GrpcLibraryInitializer init_;
+} initialize_stuff;
+
+static void BM_CreateDestroyCpp(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    CompletionQueue cq;
+  }
+}
+BENCHMARK(BM_CreateDestroyCpp);
+
+static void BM_CreateDestroyCore(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    grpc_completion_queue_destroy(grpc_completion_queue_create(NULL));
+  }
+}
+BENCHMARK(BM_CreateDestroyCore);
+
+static void DoneWithCompletionOnStack(grpc_exec_ctx* exec_ctx, void* arg,
+                                      grpc_cq_completion* completion) {}
+
+class DummyTag final : public CompletionQueueTag {
+ public:
+  bool FinalizeResult(void** tag, bool* status) override { return true; }
+};
+
+static void BM_Pass1Cpp(benchmark::State& state) {
+  CompletionQueue cq;
+  grpc_completion_queue* c_cq = cq.cq();
+  while (state.KeepRunning()) {
+    grpc_cq_completion completion;
+    DummyTag dummy_tag;
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_cq_begin_op(c_cq, &dummy_tag);
+    grpc_cq_end_op(&exec_ctx, c_cq, &dummy_tag, GRPC_ERROR_NONE,
+                   DoneWithCompletionOnStack, NULL, &completion);
+    grpc_exec_ctx_finish(&exec_ctx);
+    void* tag;
+    bool ok;
+    cq.Next(&tag, &ok);
+  }
+}
+BENCHMARK(BM_Pass1Cpp);
+
+static void BM_Pass1Core(benchmark::State& state) {
+  grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
+  gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  while (state.KeepRunning()) {
+    grpc_cq_completion completion;
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_cq_begin_op(cq, NULL);
+    grpc_cq_end_op(&exec_ctx, cq, NULL, GRPC_ERROR_NONE,
+                   DoneWithCompletionOnStack, NULL, &completion);
+    grpc_exec_ctx_finish(&exec_ctx);
+    grpc_completion_queue_next(cq, deadline, NULL);
+  }
+}
+BENCHMARK(BM_Pass1Core);
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json
index 0d5aeb233c4..8c3c1c9d9e2 100644
--- a/tools/run_tests/generated/sources_and_headers.json
+++ b/tools/run_tests/generated/sources_and_headers.json
@@ -2363,6 +2363,26 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "benchmark", 
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc++", 
+      "grpc++_test_util", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "is_filegroup": false, 
+    "language": "c++", 
+    "name": "bm_cq", 
+    "src": [
+      "test/cpp/microbenchmarks/bm_cq.cc"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "benchmark", 
diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json
index c193caa26f0..4aa8917e51e 100644
--- a/tools/run_tests/generated/tests.json
+++ b/tools/run_tests/generated/tests.json
@@ -2469,6 +2469,28 @@
       "windows"
     ]
   }, 
+  {
+    "args": [
+      "--benchmark_min_time=0"
+    ], 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ], 
+    "cpu_cost": 1.0, 
+    "exclude_configs": [], 
+    "exclude_iomgrs": [], 
+    "flaky": false, 
+    "gtest": false, 
+    "language": "c++", 
+    "name": "bm_cq", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix"
+    ]
+  }, 
   {
     "args": [
       "--benchmark_min_time=0"

From 5752570b025befb68bbc1c9586bdd53da43bae55 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Tue, 14 Feb 2017 21:58:46 -0800
Subject: [PATCH 03/10] Fix memory leak

---
 test/cpp/microbenchmarks/bm_cq.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc
index 502320d6dc3..195dcef3abd 100644
--- a/test/cpp/microbenchmarks/bm_cq.cc
+++ b/test/cpp/microbenchmarks/bm_cq.cc
@@ -109,6 +109,7 @@ static void BM_Pass1Core(benchmark::State& state) {
     grpc_exec_ctx_finish(&exec_ctx);
     grpc_completion_queue_next(cq, deadline, NULL);
   }
+  grpc_completion_queue_destroy(cq);
 }
 BENCHMARK(BM_Pass1Core);
 

From 7848b44e960168f70047555cea14530950547192 Mon Sep 17 00:00:00 2001
From: Craig Tiller <ctiller@google.com>
Date: Wed, 15 Feb 2017 08:36:19 -0800
Subject: [PATCH 04/10] Update microbenchmarking framework for new benchmark

---
 tools/run_tests/run_microbenchmark.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py
index d748e5a0ce4..4da35bc1424 100755
--- a/tools/run_tests/run_microbenchmark.py
+++ b/tools/run_tests/run_microbenchmark.py
@@ -173,7 +173,7 @@ argp.add_argument('-c', '--collect',
                   default=sorted(collectors.keys()),
                   help='Which collectors should be run against each benchmark')
 argp.add_argument('-b', '--benchmarks',
-                  default=['bm_fullstack', 'bm_closure'],
+                  default=['bm_fullstack', 'bm_closure', 'bm_cq'],
                   nargs='+',
                   type=str,
                   help='Which microbenchmarks should be run')

From 6da1e85255b921b88f0ff9a21edcbbd739c6044a Mon Sep 17 00:00:00 2001
From: yang-g <yangg@google.com>
Date: Thu, 16 Feb 2017 10:34:06 -0800
Subject: [PATCH 05/10] Contain the links for the pending handshake managers
 inside handshake_manager

---
 .../transport/chttp2/server/chttp2_server.c   | 57 +++----------------
 src/core/lib/channel/handshaker.c             | 37 ++++++++++++
 src/core/lib/channel/handshaker.h             | 16 ++++++
 3 files changed, 61 insertions(+), 49 deletions(-)

diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c
index ae2c3838ed9..0fc180d52f1 100644
--- a/src/core/ext/transport/chttp2/server/chttp2_server.c
+++ b/src/core/ext/transport/chttp2/server/chttp2_server.c
@@ -55,11 +55,6 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct pending_handshake_manager_node {
-  grpc_handshake_manager *handshake_mgr;
-  struct pending_handshake_manager_node *next;
-} pending_handshake_manager_node;
-
 typedef struct {
   grpc_server *server;
   grpc_tcp_server *tcp_server;
@@ -68,7 +63,7 @@ typedef struct {
   bool shutdown;
   grpc_closure tcp_server_shutdown_complete;
   grpc_closure *server_destroy_listener_done;
-  pending_handshake_manager_node *pending_handshake_mgrs;
+  grpc_handshake_manager *pending_handshake_mgrs;
 } server_state;
 
 typedef struct {
@@ -78,44 +73,6 @@ typedef struct {
   grpc_handshake_manager *handshake_mgr;
 } server_connection_state;
 
-static void pending_handshake_manager_add_locked(
-    server_state *state, grpc_handshake_manager *handshake_mgr) {
-  pending_handshake_manager_node *node = gpr_malloc(sizeof(*node));
-  node->handshake_mgr = handshake_mgr;
-  node->next = state->pending_handshake_mgrs;
-  state->pending_handshake_mgrs = node;
-}
-
-static void pending_handshake_manager_remove_locked(
-    server_state *state, grpc_handshake_manager *handshake_mgr) {
-  pending_handshake_manager_node **prev_node = &state->pending_handshake_mgrs;
-  for (pending_handshake_manager_node *node = state->pending_handshake_mgrs;
-       node != NULL; node = node->next) {
-    if (node->handshake_mgr == handshake_mgr) {
-      *prev_node = node->next;
-      gpr_free(node);
-      break;
-    }
-    prev_node = &node->next;
-  }
-}
-
-static void pending_handshake_manager_shutdown_locked(grpc_exec_ctx *exec_ctx,
-                                                      server_state *state,
-                                                      grpc_error *why) {
-  pending_handshake_manager_node *prev_node = NULL;
-  for (pending_handshake_manager_node *node = state->pending_handshake_mgrs;
-       node != NULL; node = node->next) {
-    grpc_handshake_manager_shutdown(exec_ctx, node->handshake_mgr,
-                                    GRPC_ERROR_REF(why));
-    gpr_free(prev_node);
-    prev_node = node;
-  }
-  gpr_free(prev_node);
-  state->pending_handshake_mgrs = NULL;
-  GRPC_ERROR_UNREF(why);
-}
-
 static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
                               grpc_error *error) {
   grpc_handshaker_args *args = arg;
@@ -153,8 +110,9 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
       grpc_channel_args_destroy(exec_ctx, args->args);
     }
   }
-  pending_handshake_manager_remove_locked(connection_state->server_state,
-                                          connection_state->handshake_mgr);
+  grpc_handshake_manager_pending_list_remove(
+      &connection_state->server_state->pending_handshake_mgrs,
+      connection_state->handshake_mgr);
   gpr_mu_unlock(&connection_state->server_state->mu);
   grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
   grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp_server);
@@ -174,7 +132,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
     return;
   }
   grpc_handshake_manager *handshake_mgr = grpc_handshake_manager_create();
-  pending_handshake_manager_add_locked(state, handshake_mgr);
+  grpc_handshake_manager_pending_list_add(&state->pending_handshake_mgrs,
+                                          handshake_mgr);
   gpr_mu_unlock(&state->mu);
   grpc_tcp_server_ref(state->tcp_server);
   server_connection_state *connection_state =
@@ -213,8 +172,8 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_mu_lock(&state->mu);
   grpc_closure *destroy_done = state->server_destroy_listener_done;
   GPR_ASSERT(state->shutdown);
-  pending_handshake_manager_shutdown_locked(exec_ctx, state,
-                                            GRPC_ERROR_REF(error));
+  grpc_handshake_manager_pending_list_shutdown_all(
+      exec_ctx, state->pending_handshake_mgrs, GRPC_ERROR_REF(error));
   gpr_mu_unlock(&state->mu);
   // Flush queued work before destroying handshaker factory, since that
   // may do a synchronous unref.
diff --git a/src/core/lib/channel/handshaker.c b/src/core/lib/channel/handshaker.c
index 5bed2d041d8..82c361c7ef6 100644
--- a/src/core/lib/channel/handshaker.c
+++ b/src/core/lib/channel/handshaker.c
@@ -92,6 +92,10 @@ struct grpc_handshake_manager {
   void* user_data;
   // Handshaker args.
   grpc_handshaker_args args;
+  // Links to the previous and next managers in a list of all pending handshakes
+  // Used at server side only.
+  grpc_handshake_manager* prev;
+  grpc_handshake_manager* next;
 };
 
 grpc_handshake_manager* grpc_handshake_manager_create() {
@@ -102,6 +106,39 @@ grpc_handshake_manager* grpc_handshake_manager_create() {
   return mgr;
 }
 
+void grpc_handshake_manager_pending_list_add(grpc_handshake_manager** head,
+                                             grpc_handshake_manager* mgr) {
+  GPR_ASSERT(mgr->prev == NULL);
+  GPR_ASSERT(mgr->next == NULL);
+  mgr->next = *head;
+  if (*head) {
+    (*head)->prev = mgr;
+  }
+  *head = mgr;
+}
+
+void grpc_handshake_manager_pending_list_remove(grpc_handshake_manager** head,
+                                                grpc_handshake_manager* mgr) {
+  if (mgr->next != NULL) {
+    mgr->next->prev = mgr->prev;
+  }
+  if (mgr->prev != NULL) {
+    mgr->prev->next = mgr->next;
+  } else {
+    GPR_ASSERT(*head == mgr);
+    *head = mgr->next;
+  }
+}
+
+void grpc_handshake_manager_pending_list_shutdown_all(
+    grpc_exec_ctx* exec_ctx, grpc_handshake_manager* head, grpc_error* why) {
+  while (head != NULL) {
+    grpc_handshake_manager_shutdown(exec_ctx, head, GRPC_ERROR_REF(why));
+    head = head->next;
+  }
+  GRPC_ERROR_UNREF(why);
+}
+
 static bool is_power_of_2(size_t n) { return (n & (n - 1)) == 0; }
 
 void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h
index a8e3692add4..5f97c3fc736 100644
--- a/src/core/lib/channel/handshaker.h
+++ b/src/core/lib/channel/handshaker.h
@@ -163,4 +163,20 @@ void grpc_handshake_manager_do_handshake(
     gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor,
     grpc_iomgr_cb_func on_handshake_done, void* user_data);
 
+/// Add \a mgr to the server side list of all pending handshake managers, the
+/// list starts with \a *head.
+// Not thread-safe. Caller needs to synchronize.
+void grpc_handshake_manager_pending_list_add(grpc_handshake_manager** head,
+                                             grpc_handshake_manager* mgr);
+
+/// Remove \a mgr from the server side list of all pending handshake managers.
+// Not thread-safe. Caller needs to synchronize.
+void grpc_handshake_manager_pending_list_remove(grpc_handshake_manager** head,
+                                                grpc_handshake_manager* mgr);
+
+/// Shutdown all pending handshake managers on the server side.
+// Not thread-safe. Caller needs to synchronize.
+void grpc_handshake_manager_pending_list_shutdown_all(
+    grpc_exec_ctx* exec_ctx, grpc_handshake_manager* head, grpc_error* why);
+
 #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H */

From e9932d4744e3e72691c2633ae752d8b9d291b26c Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Fri, 17 Feb 2017 16:44:30 -0800
Subject: [PATCH 06/10] add gflags to bazel

---
 WORKSPACE                                  | 18 ++++++++++++++----
 third_party/gflags                         |  2 +-
 tools/run_tests/sanity/check_submodules.sh |  2 +-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 4f90f06d881..9b79d04cde6 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -33,26 +33,36 @@ bind(
     actual = "@submodule_gtest//:gtest",
 )
 
+bind(
+    name = "gflags",
+    actual = "@com_github_gflags_gflags//:gflags",
+)
+
 new_local_repository(
     name = "submodule_boringssl",
-    path = "third_party/boringssl-with-bazel",
     build_file = "third_party/boringssl-with-bazel/BUILD",
+    path = "third_party/boringssl-with-bazel",
 )
 
 new_local_repository(
     name = "submodule_zlib",
-    path = "third_party/zlib",
     build_file = "third_party/zlib.BUILD",
+    path = "third_party/zlib",
 )
 
 new_local_repository(
     name = "submodule_protobuf",
-    path = "third_party/protobuf",
     build_file = "third_party/protobuf/BUILD",
+    path = "third_party/protobuf",
 )
 
 new_local_repository(
     name = "submodule_gtest",
-    path = "third_party/googletest",
     build_file = "third_party/gtest.BUILD",
+    path = "third_party/googletest",
+)
+
+local_repository(
+    name = "com_github_gflags_gflags",
+    path = "third_party/gflags",
 )
diff --git a/third_party/gflags b/third_party/gflags
index f8a0efe03aa..30dbc81fb5f 160000
--- a/third_party/gflags
+++ b/third_party/gflags
@@ -1 +1 @@
-Subproject commit f8a0efe03aa69b3336d8e228b37d4ccb17324b88
+Subproject commit 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e
diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh
index 0b68319d290..cfe4e2731c0 100755
--- a/tools/run_tests/sanity/check_submodules.sh
+++ b/tools/run_tests/sanity/check_submodules.sh
@@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules
  44c25c892a6229b20db7cd9dc05584ea865896de third_party/benchmark (v0.1.0-343-g44c25c8)
  78684e5b222645828ca302e56b40b9daff2b2d27 third_party/boringssl (78684e5)
  886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7)
- f8a0efe03aa69b3336d8e228b37d4ccb17324b88 third_party/gflags (v2.2.0)
+ 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0)
  c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0)
  a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0-alpha-1)
  bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917)

From 30192a35cf4b9a7e109a78c7e26201b7541a7e22 Mon Sep 17 00:00:00 2001
From: Yong Ni <yongni@google.com>
Date: Tue, 21 Feb 2017 12:13:41 -0800
Subject: [PATCH 07/10] Breaking grpc_interop into multiple Kokoro jobs

---
 ...op.cfg => grpc_interop_badserver_java.cfg} |  5 +--
 ...erop.sh => grpc_interop_badserver_java.sh} |  5 +--
 .../linux/grpc_interop_badserver_python.cfg   | 40 ++++++++++++++++++
 .../linux/grpc_interop_badserver_python.sh    | 41 +++++++++++++++++++
 .../linux/grpc_interop_tocloud.cfg            | 40 ++++++++++++++++++
 .../internal_ci/linux/grpc_interop_tocloud.sh | 40 ++++++++++++++++++
 6 files changed, 165 insertions(+), 6 deletions(-)
 rename tools/internal_ci/linux/{grpc_interop.cfg => grpc_interop_badserver_java.cfg} (94%)
 rename tools/internal_ci/linux/{grpc_interop.sh => grpc_interop_badserver_java.sh} (85%)
 mode change 100755 => 100644
 create mode 100644 tools/internal_ci/linux/grpc_interop_badserver_python.cfg
 create mode 100644 tools/internal_ci/linux/grpc_interop_badserver_python.sh
 create mode 100644 tools/internal_ci/linux/grpc_interop_tocloud.cfg
 create mode 100644 tools/internal_ci/linux/grpc_interop_tocloud.sh

diff --git a/tools/internal_ci/linux/grpc_interop.cfg b/tools/internal_ci/linux/grpc_interop_badserver_java.cfg
similarity index 94%
rename from tools/internal_ci/linux/grpc_interop.cfg
rename to tools/internal_ci/linux/grpc_interop_badserver_java.cfg
index 9259faf34d5..e521b085c5e 100644
--- a/tools/internal_ci/linux/grpc_interop.cfg
+++ b/tools/internal_ci/linux/grpc_interop_badserver_java.cfg
@@ -1,4 +1,3 @@
-#!/bin/bash
 # Copyright 2017, Google Inc.
 # All rights reserved.
 #
@@ -31,11 +30,11 @@
 # Config file for the internal CI (in protobuf text format)
 
 # Location of the continuous shell script in repository.
-build_file: "grpc/tools/internal_ci/linux/grpc_interop.sh"
+build_file: "grpc/tools/internal_ci/linux/grpc_interop_badserver_java.sh"
 # grpc_interop tests can take 6+ hours to complete.
 timeout_mins: 480
 action {
   define_artifacts {
-    regex: "**/sponge_log.xml"
+    regex: "**/report.xml"
   }
 }
diff --git a/tools/internal_ci/linux/grpc_interop.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh
old mode 100755
new mode 100644
similarity index 85%
rename from tools/internal_ci/linux/grpc_interop.sh
rename to tools/internal_ci/linux/grpc_interop_badserver_java.sh
index 68bb4199e3d..0985e657c63
--- a/tools/internal_ci/linux/grpc_interop.sh
+++ b/tools/internal_ci/linux/grpc_interop_badserver_java.sh
@@ -37,6 +37,5 @@ cd $(dirname $0)/../../..
 
 git submodule update --init
 
-tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop -t -j 12 $@ || FAILED="true"
-tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@ || FAILED="true"
-tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@ || FAILED="true"
+tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@
+
diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.cfg b/tools/internal_ci/linux/grpc_interop_badserver_python.cfg
new file mode 100644
index 00000000000..940f760e973
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_badserver_python.cfg
@@ -0,0 +1,40 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_interop_badserver_python.sh"
+# grpc_interop tests can take 6+ hours to complete.
+timeout_mins: 480
+action {
+  define_artifacts {
+    regex: "**/report.xml"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh
new file mode 100644
index 00000000000..3fff537d2b7
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_badserver_python.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+export LANG=en_US.UTF-8
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+git submodule update --init
+
+tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@
+
diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg
new file mode 100644
index 00000000000..2b536446c1b
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg
@@ -0,0 +1,40 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_interop_tocloud.sh"
+# grpc_interop tests can take 6+ hours to complete.
+timeout_mins: 480
+action {
+  define_artifacts {
+    regex: "**/report.xml"
+  }
+}
diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh
new file mode 100644
index 00000000000..572001d9441
--- /dev/null
+++ b/tools/internal_ci/linux/grpc_interop_tocloud.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+export LANG=en_US.UTF-8
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+git submodule update --init
+
+tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop -t -j 12 $@

From facc5c72ea27afd83af95f059b339bc8893a03e2 Mon Sep 17 00:00:00 2001
From: Yong Ni <yongni@google.com>
Date: Tue, 21 Feb 2017 12:19:48 -0800
Subject: [PATCH 08/10] Tue Feb 21 12:19:48 PST 2017

---
 tools/internal_ci/linux/grpc_interop_badserver_java.sh   | 0
 tools/internal_ci/linux/grpc_interop_badserver_python.sh | 0
 tools/internal_ci/linux/grpc_interop_tocloud.sh          | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tools/internal_ci/linux/grpc_interop_badserver_java.sh
 mode change 100644 => 100755 tools/internal_ci/linux/grpc_interop_badserver_python.sh
 mode change 100644 => 100755 tools/internal_ci/linux/grpc_interop_tocloud.sh

diff --git a/tools/internal_ci/linux/grpc_interop_badserver_java.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh
old mode 100644
new mode 100755
diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh
old mode 100644
new mode 100755
diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh
old mode 100644
new mode 100755

From 84b4214f1a968682b2ad5ac226a91c9c8d7ca37e Mon Sep 17 00:00:00 2001
From: Muxi Yan <mxyan@google.com>
Date: Tue, 21 Feb 2017 13:46:10 -0800
Subject: [PATCH 09/10] Fix flakiness in testPacketCoalescing

---
 .../tests/CronetUnitTests/CronetUnitTests.m   | 82 ++++++++++---------
 1 file changed, 43 insertions(+), 39 deletions(-)

diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
index 5e3c59f8b34..e97f3d2d1a9 100644
--- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
+++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m
@@ -321,44 +321,8 @@ unsigned int parse_h2_length(const char *field) {
   grpc_metadata_array_init(&request_metadata_recv);
   grpc_call_details_init(&call_details);
 
-  memset(ops, 0, sizeof(ops));
-  op = ops;
-  op->op = GRPC_OP_SEND_INITIAL_METADATA;
-  op->data.send_initial_metadata.count = 2;
-  op->data.send_initial_metadata.metadata = meta_c;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  op->op = GRPC_OP_SEND_MESSAGE;
-  op->data.send_message.send_message = request_payload;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  op->op = GRPC_OP_RECV_INITIAL_METADATA;
-  op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  op->op = GRPC_OP_RECV_MESSAGE;
-  op->data.recv_message.recv_message = &response_payload_recv;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
-  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
-  op->data.recv_status_on_client.status = &status;
-  op->data.recv_status_on_client.status_details = &details;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void *)1, NULL);
-  GPR_ASSERT(GRPC_CALL_OK == error);
-
-  __weak XCTestExpectation *expectation = [self expectationWithDescription:@"Coalescing"];
+  __weak XCTestExpectation *expectation =
+      [self expectationWithDescription:@"Coalescing"];
 
   dispatch_async(
       dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
@@ -425,6 +389,46 @@ unsigned int parse_h2_length(const char *field) {
         [expectation fulfill];
       });
 
+  // Guarantees that server is listening to the port before client connects.
+  sleep(1);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 2;
+  op->data.send_initial_metadata.metadata = meta_c;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_SEND_MESSAGE;
+  op->data.send_message.send_message = request_payload;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_INITIAL_METADATA;
+  op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_MESSAGE;
+  op->data.recv_message.recv_message = &response_payload_recv;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
+  op->data.recv_status_on_client.status = &status;
+  op->data.recv_status_on_client.status_details = &details;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), (void *)1, NULL);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
   CQ_EXPECT_COMPLETION(cqv, (void *)1, 1);
   cq_verify(cqv);
 
@@ -445,7 +449,7 @@ unsigned int parse_h2_length(const char *field) {
   grpc_completion_queue_shutdown(cq);
   drain_cq(cq);
   grpc_completion_queue_destroy(cq);
-  
+
   [self waitForExpectationsWithTimeout:4 handler:nil];
 }
 

From 388b24d5a13e61b08dbd411c1978982d1df536ec Mon Sep 17 00:00:00 2001
From: David Garcia Quintas <dgq@google.com>
Date: Tue, 21 Feb 2017 14:58:32 -0800
Subject: [PATCH 10/10] Added gflags dependency to test_config

---
 test/cpp/util/BUILD | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD
index 78eca8c3bd2..f3cdc58986a 100644
--- a/test/cpp/util/BUILD
+++ b/test/cpp/util/BUILD
@@ -37,14 +37,17 @@ cc_library(
     hdrs = [
         "test_config.h",
     ],
-    deps = ["//:gpr"],
     visibility = ["//test:__subpackages__"],
+    deps = [
+        "//:gpr",
+        "//external:gflags",
+    ],
 )
 
 cc_library(
     name = "test_util",
     srcs = [
-#        "test/cpp/end2end/test_service_impl.cc",
+        #        "test/cpp/end2end/test_service_impl.cc",
         "byte_buffer_proto_helper.cc",
         "create_test_channel.cc",
         "string_ref_helper.cc",
@@ -58,6 +61,10 @@ cc_library(
         "subprocess.h",
         "test_credentials_provider.h",
     ],
-    deps = ["//test/core/util:gpr_test_util", "//:grpc++", "//test/core/end2end:ssl_test_data"],
     visibility = ["//test:__subpackages__"],
+    deps = [
+        "//:grpc++",
+        "//test/core/end2end:ssl_test_data",
+        "//test/core/util:gpr_test_util",
+    ],
 )